Merge remote-tracking branch 'upstream/master'

.github/workflows/ci.yml

@@ -29,7 +29,32 @@ jobs:
         github_token: ${{ secrets.GITHUB_TOKEN }}
     - run: nix flake show --all-systems --json
 
+  pre-commit-checks:
+    name: pre-commit checks
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/install-nix-action
+        with:
+          dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
+          extra_nix_config: experimental-features = nix-command flakes
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: ./ci/gha/tests/pre-commit-checks
+
+  basic-checks:
+    name: aggregate basic checks
+    if: ${{ always() }}
+    runs-on: ubuntu-24.04
+    needs: [pre-commit-checks, eval]
+    steps:
+      - name: Exit with any errors
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+        run: |
+          exit 1
+
   tests:
+    needs: basic-checks
     strategy:
       fail-fast: false
       matrix:
@@ -214,6 +239,7 @@ jobs:
         docker push $IMAGE_ID:master
 
   vm_tests:
+    needs: basic-checks
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v5

COPYING

@@ -2,7 +2,7 @@
                        Version 2.1, February 1999
 
  Copyright (C) 1991, 1999 Free Software Foundation, Inc.
- 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -484,8 +484,7 @@ convey the exclusion of warranty; and each file should have at least the
     Lesser General Public License for more details.
 
     You should have received a copy of the GNU Lesser General Public
-    License along with this library; if not, write to the Free Software
+    License along with this library; if not, see <https://www.gnu.org/licenses/>.
-    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -496,9 +495,7 @@ necessary.  Here is a sample; alter the names:
   Yoyodyne, Inc., hereby disclaims all copyright interest in the
   library `Frob' (a library for tweaking knobs) written by James Random Hacker.
 
-  <signature of Ty Coon>, 1 April 1990
+  <signature of Moe Ghoul>, 1 April 1990
-  Ty Coon, President of Vice
+  Moe Ghoul, President of Vice
 
 That's all there is to it!
-
-

ci/gha/tests/default.nix

@@ -24,16 +24,7 @@ let
   enableSanitizersLayer = finalAttrs: prevAttrs: {
     mesonFlags =
       (prevAttrs.mesonFlags or [ ])
-      ++ [
+      ++ [ (lib.mesonOption "b_sanitize" "address,undefined") ]
-        # Run all tests with UBSAN enabled. Running both with ubsan and
-        # without doesn't seem to have much immediate benefit for doubling
-        # the GHA CI workaround.
-        #
-        # TODO: Work toward enabling "address,undefined" if it seems feasible.
-        # This would maybe require dropping Boost coroutines and ignoring intentional
-        # memory leaks with detect_leaks=0.
-        (lib.mesonOption "b_sanitize" "undefined")
-      ]
       ++ (lib.optionals stdenv.cc.isClang [
         # https://www.github.com/mesonbuild/meson/issues/764
         (lib.mesonBool "b_lundef" false)
@@ -71,8 +62,12 @@ rec {
   nixComponentsInstrumented = nixComponents.overrideScope (
     final: prev: {
       nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; };
+      # Boehm is incompatible with ASAN.
+      nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; };
 
       mesonComponentOverrides = lib.composeManyExtensions componentOverrides;
+      # Unclear how to make Perl bindings work with a dynamically linked ASAN.
+      nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings;
     }
   );
 

ci/gha/tests/pre-commit-checks

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+system=$(nix eval --raw --impure --expr builtins.currentSystem)
+
+echo "::group::Running pre-commit checks"
+
+if nix build ".#checks.$system.pre-commit" -L; then
+    echo "::endgroup::"
+    exit 0
+fi
+
+echo "::error ::Changes do not pass pre-commit checks"
+
+cat <<EOF
+The code isn't formatted or doesn't pass lints. You can run pre-commit locally with:
+
+    nix develop -c ./maintainers/format.sh
+EOF
+
+echo "::endgroup::"
+
+exit 1

doc/manual/meson.build

@@ -15,6 +15,7 @@ pymod = import('python')
 python = pymod.find_installation('python3')
 
 nix_env_for_docs = {
+  'ASAN_OPTIONS' : 'abort_on_error=1:print_summary=1:detect_leaks=0',
   'HOME' : '/dummy',
   'NIX_CONF_DIR' : '/dummy',
   'NIX_SSL_CERT_FILE' : '/dummy/no-ca-bundle.crt',

doc/manual/rl-next/c-api-byidx.md

@@ -0,0 +1,7 @@
+---
+synopsis: "C API: `nix_get_attr_name_byidx`, `nix_get_attr_byidx` take a `nix_value *` instead of `const nix_value *`"
+prs: [13987]
+---
+
+In order to accommodate a more optimized internal representation of attribute set merges these functions require
+a mutable `nix_value *` that might be modified on access. This does *not* break the ABI of these functions.

doc/manual/rl-next/c-api-lazy-accessors.md

@@ -0,0 +1,16 @@
+---
+synopsis: "C API: Add lazy attribute and list item accessors"
+prs: [14030]
+---
+
+The C API now includes lazy accessor functions for retrieving values from lists and attribute sets without forcing evaluation:
+
+- `nix_get_list_byidx_lazy()` - Get a list element without forcing its evaluation
+- `nix_get_attr_byname_lazy()` - Get an attribute value by name without forcing evaluation
+- `nix_get_attr_byidx_lazy()` - Get an attribute by index without forcing evaluation
+
+These functions are useful when forwarding unevaluated sub-values to other lists, attribute sets, or function calls. They allow more efficient handling of Nix values by deferring evaluation until actually needed.
+
+Additionally, bounds checking has been improved for all `_byidx` functions to properly validate indices before access, preventing potential out-of-bounds errors.
+
+The documentation for `NIX_ERR_KEY` error handling has also been clarified to specify when this error code is returned.

doc/manual/rl-next/cached-substituted-inputs.md

@@ -0,0 +1,10 @@
+---
+synopsis: "Substituted flake inputs are no longer re-copied to the store"
+prs: [14041]
+---
+
+Since 2.25, Nix would fail to store a cache entry for substituted flake inputs,
+which in turn would cause them to be re-copied to the store on initial
+evaluation. Caching these inputs results in a near doubling of a performance in
+some cases — especially on I/O-bound machines and when using commands that
+fetch many inputs, like `nix flake archive/prefetch-inputs`

doc/manual/rl-next/derivation-json.md

@@ -0,0 +1,15 @@
+---
+synopsis: Derivation JSON format now uses store path basenames (no store dir) only
+prs: [13980]
+issues: [13570]
+---
+
+Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell), has shown that the use of the store dir in JSON formats is an impediment to systematic JSON formats,
+because it requires the serializer/deserializer to take an extra paramater (the store dir).
+
+We ultimately want to rectify this issue with all (non-stable, able to be changed) JSON formats.
+To start with, we are changing the JSON format for derivations because the `nix derivation` commands are
+--- in addition to being formally unstable
+--- less widely used than other unstable commands.
+
+See the documentation on the [JSON format for derivations](@docroot@/protocols/json/derivation.md) for further details.

doc/manual/source/command-ref/meson.build

@@ -2,6 +2,7 @@ xp_features_json = custom_target(
   command : [ nix, '__dump-xp-features' ],
   capture : true,
   output : 'xp-features.json',
+  env : nix_env_for_docs,
 )
 
 experimental_features_shortlist_md = custom_target(

doc/manual/source/development/building.md

@@ -23,7 +23,7 @@ $ nix-shell
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages
+$ nix-shell --attr devShells.x86_64-linux.native-clangStdenv
 ```
 
 > **Note**

doc/manual/source/development/debugging.md

@@ -24,6 +24,19 @@ It is also possible to build without debugging for faster build:
 
 (The first line is needed because `fortify` hardening requires at least some optimization.)
 
+## Building Nix with sanitizers
+
+Nix can be built with [Address](https://clang.llvm.org/docs/AddressSanitizer.html) and
+[UB](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html) sanitizers using LLVM
+or GCC. This is useful when debugging memory corruption issues.
+
+```console
+[nix-shell]$ export mesonBuildType=debugoptimized
+[nix-shell]$ appendToVar mesonFlags "-Dlibexpr:gc=disabled" # Disable Boehm
+[nix-shell]$ appendToVar mesonFlags "-Dbindings=false" # Disable nix-perl
+[nix-shell]$ appendToVar mesonFlags "-Db_sanitize=address,undefined"
+```
+
 ## Debugging the Nix Binary
 
 Obtain your preferred debugger within the development shell:

doc/manual/source/development/meson.build

@@ -7,5 +7,6 @@ experimental_feature_descriptions_md = custom_target(
     xp_features_json,
   ],
   capture : true,
+  env : nix_env_for_docs,
   output : 'experimental-feature-descriptions.md',
 )

doc/manual/source/language/builtins-prefix.md

@@ -5,12 +5,28 @@ All built-ins are available through the global [`builtins`](#builtins-builtins)
 
 Some built-ins are also exposed directly in the global scope:
 
-<!-- TODO(@rhendric, #10970): this list is incomplete -->
-
 - [`derivation`](#builtins-derivation)
-- [`import`](#builtins-import)
+- `derivationStrict`
 - [`abort`](#builtins-abort)
+- [`baseNameOf`](#builtins-baseNameOf)
+- [`break`](#builtins-break)
+- [`dirOf`](#builtins-dirOf)
+- [`false`](#builtins-false)
+- [`fetchGit`](#builtins-fetchGit)
+- `fetchMercurial`
+- [`fetchTarball`](#builtins-fetchTarball)
+- [`fetchTree`](#builtins-fetchTree)
+- [`fromTOML`](#builtins-fromTOML)
+- [`import`](#builtins-import)
+- [`isNull`](#builtins-isNull)
+- [`map`](#builtins-map)
+- [`null`](#builtins-null)
+- [`placeholder`](#builtins-placeholder)
+- [`removeAttrs`](#builtins-removeAttrs)
+- `scopedImport`
 - [`throw`](#builtins-throw)
+- [`toString`](#builtins-toString)
+- [`true`](#builtins-true)
 
 <dl>
   <dt id="builtins-derivation"><a href="#builtins-derivation"><code>derivation <var>attrs</var></code></a></dt>

doc/manual/source/protocols/json/derivation.md

@@ -14,6 +14,21 @@ is a JSON object with the following fields:
   The name of the derivation.
   This is used when calculating the store paths of the derivation's outputs.
 
+* `version`:
+  Must be `3`.
+  This is a guard that allows us to continue evolving this format.
+  The choice of `3` is fairly arbitrary, but corresponds to this informal version:
+
+  - Version 0: A-Term format
+
+  - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format.
+
+  - Version 2: Separate `method` and `hashAlgo` fields in output specs
+
+  - Verison 3: Drop store dir from store paths, just include base name.
+
+  Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
+
 * `outputs`:
   Information about the output paths of the derivation.
   This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
@@ -52,7 +67,6 @@ is a JSON object with the following fields:
   > ```json
   > "outputs": {
   >   "out": {
-  >     "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
   >     "method": "nar",
   >     "hashAlgo": "sha256",
   >     "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
@@ -63,6 +77,15 @@ is a JSON object with the following fields:
 * `inputSrcs`:
   A list of store paths on which this derivation depends.
 
+  > **Example**
+  >
+  > ```json
+  > "inputSrcs": [
+  >   "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
+  >   "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
+  > ]
+  > ```
+
 * `inputDrvs`:
   A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations.
 
@@ -70,8 +93,8 @@ is a JSON object with the following fields:
   >
   > ```json
   > "inputDrvs": {
-  >   "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
+  >   "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
-  >   "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
+  >   "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
   > }
   > ```
 

maintainers/flake-module.nix

@@ -106,63 +106,6 @@
             enable = true;
             excludes = [
               # We haven't linted these files yet
-              ''^config/install-sh$''
-              ''^misc/bash/completion\.sh$''
-              ''^misc/fish/completion\.fish$''
-              ''^misc/zsh/completion\.zsh$''
-              ''^scripts/create-darwin-volume\.sh$''
-              ''^scripts/install-darwin-multi-user\.sh$''
-              ''^scripts/install-multi-user\.sh$''
-              ''^scripts/install-systemd-multi-user\.sh$''
-              ''^src/nix/get-env\.sh$''
-              ''^tests/functional/ca/build-dry\.sh$''
-              ''^tests/functional/ca/build-with-garbage-path\.sh$''
-              ''^tests/functional/ca/common\.sh$''
-              ''^tests/functional/ca/concurrent-builds\.sh$''
-              ''^tests/functional/ca/eval-store\.sh$''
-              ''^tests/functional/ca/gc\.sh$''
-              ''^tests/functional/ca/import-from-derivation\.sh$''
-              ''^tests/functional/ca/new-build-cmd\.sh$''
-              ''^tests/functional/ca/nix-shell\.sh$''
-              ''^tests/functional/ca/post-hook\.sh$''
-              ''^tests/functional/ca/recursive\.sh$''
-              ''^tests/functional/ca/repl\.sh$''
-              ''^tests/functional/ca/selfref-gc\.sh$''
-              ''^tests/functional/ca/why-depends\.sh$''
-              ''^tests/functional/characterisation-test-infra\.sh$''
-              ''^tests/functional/common/vars-and-functions\.sh$''
-              ''^tests/functional/completions\.sh$''
-              ''^tests/functional/compute-levels\.sh$''
-              ''^tests/functional/config\.sh$''
-              ''^tests/functional/db-migration\.sh$''
-              ''^tests/functional/debugger\.sh$''
-              ''^tests/functional/dependencies\.builder0\.sh$''
-              ''^tests/functional/dependencies\.sh$''
-              ''^tests/functional/dump-db\.sh$''
-              ''^tests/functional/dyn-drv/build-built-drv\.sh$''
-              ''^tests/functional/dyn-drv/common\.sh$''
-              ''^tests/functional/dyn-drv/dep-built-drv\.sh$''
-              ''^tests/functional/dyn-drv/eval-outputOf\.sh$''
-              ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$''
-              ''^tests/functional/dyn-drv/recursive-mod-json\.sh$''
-              ''^tests/functional/eval-store\.sh$''
-              ''^tests/functional/export-graph\.sh$''
-              ''^tests/functional/export\.sh$''
-              ''^tests/functional/extra-sandbox-profile\.sh$''
-              ''^tests/functional/fetchClosure\.sh$''
-              ''^tests/functional/fetchGit\.sh$''
-              ''^tests/functional/fetchGitRefs\.sh$''
-              ''^tests/functional/fetchGitSubmodules\.sh$''
-              ''^tests/functional/fetchGitVerification\.sh$''
-              ''^tests/functional/fetchMercurial\.sh$''
-              ''^tests/functional/fixed\.builder1\.sh$''
-              ''^tests/functional/fixed\.builder2\.sh$''
-              ''^tests/functional/fixed\.sh$''
-              ''^tests/functional/flakes/absolute-paths\.sh$''
-              ''^tests/functional/flakes/check\.sh$''
-              ''^tests/functional/flakes/config\.sh$''
-              ''^tests/functional/flakes/flakes\.sh$''
-              ''^tests/functional/flakes/follow-paths\.sh$''
               ''^tests/functional/flakes/prefetch\.sh$''
               ''^tests/functional/flakes/run\.sh$''
               ''^tests/functional/flakes/show\.sh$''
@@ -179,29 +122,6 @@
               ''^tests/functional/install-darwin\.sh$''
               ''^tests/functional/legacy-ssh-store\.sh$''
               ''^tests/functional/linux-sandbox\.sh$''
-              ''^tests/functional/local-overlay-store/add-lower-inner\.sh$''
-              ''^tests/functional/local-overlay-store/add-lower\.sh$''
-              ''^tests/functional/local-overlay-store/bad-uris\.sh$''
-              ''^tests/functional/local-overlay-store/build-inner\.sh$''
-              ''^tests/functional/local-overlay-store/build\.sh$''
-              ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$''
-              ''^tests/functional/local-overlay-store/check-post-init\.sh$''
-              ''^tests/functional/local-overlay-store/common\.sh$''
-              ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$''
-              ''^tests/functional/local-overlay-store/delete-duplicate\.sh$''
-              ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$''
-              ''^tests/functional/local-overlay-store/delete-refs\.sh$''
-              ''^tests/functional/local-overlay-store/gc-inner\.sh$''
-              ''^tests/functional/local-overlay-store/gc\.sh$''
-              ''^tests/functional/local-overlay-store/optimise-inner\.sh$''
-              ''^tests/functional/local-overlay-store/optimise\.sh$''
-              ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$''
-              ''^tests/functional/local-overlay-store/redundant-add\.sh$''
-              ''^tests/functional/local-overlay-store/remount\.sh$''
-              ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$''
-              ''^tests/functional/local-overlay-store/stale-file-handle\.sh$''
-              ''^tests/functional/local-overlay-store/verify-inner\.sh$''
-              ''^tests/functional/local-overlay-store/verify\.sh$''
               ''^tests/functional/logging\.sh$''
               ''^tests/functional/misc\.sh$''
               ''^tests/functional/multiple-outputs\.sh$''
@@ -248,6 +168,23 @@
               ''^tests/functional/user-envs\.builder\.sh$''
               ''^tests/functional/user-envs\.sh$''
               ''^tests/functional/why-depends\.sh$''
+
+              # Content-addressed test files that use recursive-*looking* sourcing
+              # (cd .. && source <self>), causing shellcheck to loop
+              # They're small wrapper scripts with not a lot going on
+              ''^tests/functional/ca/build-delete\.sh$''
+              ''^tests/functional/ca/build-dry\.sh$''
+              ''^tests/functional/ca/eval-store\.sh$''
+              ''^tests/functional/ca/gc\.sh$''
+              ''^tests/functional/ca/import-from-derivation\.sh$''
+              ''^tests/functional/ca/multiple-outputs\.sh$''
+              ''^tests/functional/ca/new-build-cmd\.sh$''
+              ''^tests/functional/ca/nix-shell\.sh$''
+              ''^tests/functional/ca/post-hook\.sh$''
+              ''^tests/functional/ca/recursive\.sh$''
+              ''^tests/functional/ca/repl\.sh$''
+              ''^tests/functional/ca/selfref-gc\.sh$''
+              ''^tests/functional/ca/why-depends\.sh$''
             ];
           };
         };

meson.build

@@ -41,8 +41,10 @@ subproject('libexpr-c')
 subproject('libflake-c')
 subproject('libmain-c')
 
+asan_enabled = 'address' in get_option('b_sanitize')
+
 # Language Bindings
-if get_option('bindings') and not meson.is_cross_build()
+if get_option('bindings') and not meson.is_cross_build() and not asan_enabled
   subproject('perl')
 endif
 

misc/bash/completion.sh

@@ -1,3 +1,4 @@
+# shellcheck shell=bash
 function _complete_nix {
     local -a words
     local cword cur

misc/fish/completion.fish

@@ -1,3 +1,4 @@
+# shellcheck disable=all
 function _nix_complete
   # Get the current command up to a cursor.
   # - Behaves correctly even with pipes and nested in commands like env.

misc/zsh/completion.zsh

@@ -1,3 +1,4 @@
+# shellcheck disable=all
 #compdef nix
 
 function _nix() {

nix-meson-build-support/asan-options/meson.build

@@ -0,0 +1,12 @@
+asan_test_options_env = {
+  'ASAN_OPTIONS' : 'abort_on_error=1:print_summary=1:detect_leaks=0',
+}
+
+# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
+# passed when building shared libs, at least on Linux
+if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
+  'b_sanitize',
+))
+  add_project_link_arguments('-shared-libasan', language : 'cpp')
+endif
+

nix-meson-build-support/common/meson.build

@@ -5,6 +5,15 @@ if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc')
   deps_private += dependency('threads')
 endif
 
+if host_machine.system() == 'cygwin'
+  # -std=gnu on cygwin defines 'unix', which conflicts with the namespace
+  add_project_arguments(
+    '-D_POSIX_C_SOURCE=200809L',
+    '-D_GNU_SOURCE',
+    language : 'cpp',
+  )
+endif
+
 add_project_arguments(
   '-Wdeprecated-copy',
   '-Werror=suggest-override',
@@ -33,13 +42,5 @@ if cxx.get_id() == 'clang'
   add_project_arguments('-fpch-instantiate-templates', language : 'cpp')
 endif
 
-# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
-# passed when building shared libs, at least on Linux
-if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
-  'b_sanitize',
-))
-  add_project_link_arguments('-shared-libasan', language : 'cpp')
-endif
-
 # Darwin ld doesn't like "X.Y.Zpre"
-nix_soversion = meson.project_version().strip('pre')
+nix_soversion = meson.project_version().split('pre')[0]

packaging/components.nix

@@ -164,6 +164,24 @@ let
   };
 
   mesonLibraryLayer = finalAttrs: prevAttrs: {
+    preConfigure =
+      let
+        interpositionFlags = [
+          "-fno-semantic-interposition"
+          "-Wl,-Bsymbolic-functions"
+        ];
+      in
+      # NOTE: By default GCC disables interprocedular optimizations (in particular inlining) for
+      # position-independent code and thus shared libraries.
+      # Since LD_PRELOAD tricks aren't worth losing out on optimizations, we disable it for good.
+      # This is not the case for Clang, where inlining is done by default even without -fno-semantic-interposition.
+      # https://reviews.llvm.org/D102453
+      # https://fedoraproject.org/wiki/Changes/PythonNoSemanticInterpositionSpeedup
+      prevAttrs.preConfigure or ""
+      + lib.optionalString stdenv.cc.isGNU ''
+        export CFLAGS="''${CFLAGS:-} ${toString interpositionFlags}"
+        export CXXFLAGS="''${CXXFLAGS:-} ${toString interpositionFlags}"
+      '';
     outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ];
   };
 

packaging/dev-shell.nix

@@ -118,6 +118,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
         modular.pre-commit.settings.package
         (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript)
         pkgs.buildPackages.nixfmt-rfc-style
+        pkgs.buildPackages.shellcheck
         pkgs.buildPackages.gdb
       ]
       ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (

scripts/install-multi-user.sh

@@ -55,18 +55,22 @@ readonly NIX_INSTALLED_NIX="@nix@"
 readonly NIX_INSTALLED_CACERT="@cacert@"
 #readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6"
 #readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2"
-readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
+EXTRACTED_NIX_PATH="$(dirname "$0")"
+readonly EXTRACTED_NIX_PATH
 
 # allow to override identity change command
-readonly NIX_BECOME=${NIX_BECOME:-sudo}
+NIX_BECOME=${NIX_BECOME:-sudo}
+readonly NIX_BECOME
 
-readonly ROOT_HOME=~root
+ROOT_HOME=~root
+readonly ROOT_HOME
 
 if [ -t 0 ] && [ -z "${NIX_INSTALLER_YES:-}" ]; then
-    readonly IS_HEADLESS='no'
+    IS_HEADLESS='no'
 else
-    readonly IS_HEADLESS='yes'
+    IS_HEADLESS='yes'
 fi
+readonly IS_HEADLESS
 
 headless() {
     if [ "$IS_HEADLESS" = "yes" ]; then
@@ -156,6 +160,7 @@ EOF
 }
 
 nix_user_for_core() {
+    # shellcheck disable=SC2059
     printf "$NIX_BUILD_USER_NAME_TEMPLATE" "$1"
 }
 
@@ -381,10 +386,12 @@ _sudo() {
 
 # Ensure that $TMPDIR exists if defined.
 if [[ -n "${TMPDIR:-}" ]] && [[ ! -d "${TMPDIR:-}" ]]; then
+    # shellcheck disable=SC2174
     mkdir -m 0700 -p "${TMPDIR:-}"
 fi
 
-readonly SCRATCH=$(mktemp -d)
+SCRATCH=$(mktemp -d)
+readonly SCRATCH
 finish_cleanup() {
     rm -rf "$SCRATCH"
 }
@@ -677,7 +684,8 @@ create_directories() {
         # hiding behind || true, and the general state
         # should be one the user can repair once they
         # figure out where chown is...
-        local get_chr_own="$(PATH="$(getconf PATH 2>/dev/null)" command -vp chown)"
+        local get_chr_own
+        get_chr_own="$(PATH="$(getconf PATH 2>/dev/null)" command -vp chown)"
         if [[ -z "$get_chr_own" ]]; then
             get_chr_own="$(command -v chown)"
         fi
@@ -915,9 +923,11 @@ configure_shell_profile() {
         fi
 
         if [ -e "$profile_target" ]; then
-            shell_source_lines \
+            {
-                | _sudo "extend your $profile_target with nix-daemon settings" \
+                shell_source_lines
-                        tee -a "$profile_target"
+                cat "$profile_target"
+            } | _sudo "extend your $profile_target with nix-daemon settings" \
+                      tee "$profile_target"
         fi
     done
 
@@ -1013,6 +1023,7 @@ main() {
 
     # Set profile targets after OS-specific scripts are loaded
     if command -v poly_configure_default_profile_targets > /dev/null 2>&1; then
+        # shellcheck disable=SC2207
         PROFILE_TARGETS=($(poly_configure_default_profile_targets))
     else
         PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc" "/etc/bash.bashrc" "/etc/zsh/zshrc")

scripts/install-systemd-multi-user.sh

@@ -39,7 +39,7 @@ create_systemd_proxy_env() {
     vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY"
     for v in $vars; do
         if [ "x${!v:-}" != "x" ]; then
-            echo "Environment=${v}=$(escape_systemd_env ${!v})"
+            echo "Environment=${v}=$(escape_systemd_env "${!v}")"
         fi
     done
 }

src/libcmd/built-path.cc

@@ -83,12 +83,22 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) cons
 
 nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const
 {
-    return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
+    return std::visit(
+        overloaded{
+            [&](const SingleBuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); },
+            [&](const SingleBuiltPath::Built & b) { return b.toJSON(store); },
+        },
+        raw());
 }
 
 nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const
 {
-    return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
+    return std::visit(
+        overloaded{
+            [&](const BuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); },
+            [&](const BuiltPath::Built & b) { return b.toJSON(store); },
+        },
+        raw());
 }
 
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const

src/libcmd/meson.build

@@ -67,6 +67,7 @@ config_priv_h = configure_file(
 )
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'built-path.cc',

src/libcmd/repl.cc

@@ -760,7 +760,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
 
 void NixRepl::initEnv()
 {
-    env = &state->allocEnv(envSize);
+    env = &state->mem.allocEnv(envSize);
     env->up = &state->baseEnv;
     displ = 0;
     staticEnv->vars.clear();
@@ -869,14 +869,8 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
 
 Expr * NixRepl::parseString(std::string s)
 {
-    return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv);
-}
-
-void NixRepl::evalString(std::string s, Value & v)
-{
-    Expr * e;
     try {
-        e = parseString(s);
+        return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv);
     } catch (ParseError & e) {
         if (e.msg().find("unexpected end of file") != std::string::npos)
             // For parse errors on incomplete input, we continue waiting for the next line of
@@ -885,6 +879,11 @@ void NixRepl::evalString(std::string s, Value & v)
         else
             throw;
     }
+}
+
+void NixRepl::evalString(std::string s, Value & v)
+{
+    Expr * e = parseString(s);
     e->eval(*state, *env, v);
     state->forceValue(v, v.determinePos(noPos));
 }

src/libexpr-c/meson.build

@@ -28,6 +28,7 @@ deps_public_maybe_subproject = [
 subdir('nix-meson-build-support/subprojects')
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'nix_api_expr.cc',

src/libexpr-c/nix_api_expr.cc

@@ -137,7 +137,7 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto
 
 void nix_eval_state_builder_free(nix_eval_state_builder * builder)
 {
-    delete builder;
+    operator delete(builder, static_cast<std::align_val_t>(alignof(nix_eval_state_builder)));
 }
 
 nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder)
@@ -203,7 +203,7 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
 
 void nix_state_free(EvalState * state)
 {
-    delete state;
+    operator delete(state, static_cast<std::align_val_t>(alignof(EvalState)));
 }
 
 #if NIX_USE_BOEHMGC

src/libexpr-c/nix_api_value.cc

@@ -326,6 +326,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
     try {
         auto & v = check_value_in(value);
         assert(v.type() == nix::nList);
+        if (ix >= v.listSize()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "list index out of bounds");
+            return nullptr;
+        }
         auto * p = v.listView()[ix];
         nix_gc_incref(nullptr, p);
         if (p != nullptr)
@@ -335,6 +339,26 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
     NIXC_CATCH_ERRS_NULL
 }
 
+nix_value *
+nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nList);
+        if (ix >= v.listSize()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "list index out of bounds");
+            return nullptr;
+        }
+        auto * p = v.listView()[ix];
+        nix_gc_incref(nullptr, p);
+        // Note: intentionally NOT calling forceValue() to keep the element lazy
+        return as_nix_value_ptr(p);
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
 nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
 {
     if (context)
@@ -355,6 +379,27 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
     NIXC_CATCH_ERRS_NULL
 }
 
+nix_value *
+nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr) {
+            nix_gc_incref(nullptr, attr->value);
+            // Note: intentionally NOT calling forceValue() to keep the attribute lazy
+            return as_nix_value_ptr(attr->value);
+        }
+        nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
 bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
 {
     if (context)
@@ -371,13 +416,28 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS
     NIXC_CATCH_ERRS_RES(false);
 }
 
-nix_value * nix_get_attr_byidx(
+static void collapse_attrset_layer_chain_if_needed(nix::Value & v, EvalState * state)
-    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name)
+{
+    auto & attrs = *v.attrs();
+    if (attrs.isLayered()) {
+        auto bindings = state->state.buildBindings(attrs.size());
+        std::ranges::copy(attrs, std::back_inserter(bindings));
+        v.mkAttrs(bindings);
+    }
+}
+
+nix_value *
+nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_in(value);
+        collapse_attrset_layer_chain_if_needed(v, state);
+        if (i >= v.attrs()->size()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds");
+            return nullptr;
+        }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
         nix_gc_incref(nullptr, a.value);
@@ -387,13 +447,38 @@ nix_value * nix_get_attr_byidx(
     NIXC_CATCH_ERRS_NULL
 }
 
-const char *
+nix_value * nix_get_attr_byidx_lazy(
-nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i)
+    nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name)
 {
     if (context)
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_in(value);
+        collapse_attrset_layer_chain_if_needed(v, state);
+        if (i >= v.attrs()->size()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds (Nix C API contract violation)");
+            return nullptr;
+        }
+        const nix::Attr & a = (*v.attrs())[i];
+        *name = state->state.symbols[a.name].c_str();
+        nix_gc_incref(nullptr, a.value);
+        // Note: intentionally NOT calling forceValue() to keep the attribute lazy
+        return as_nix_value_ptr(a.value);
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_in(value);
+        collapse_attrset_layer_chain_if_needed(v, state);
+        if (i >= v.attrs()->size()) {
+            nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds (Nix C API contract violation)");
+            return nullptr;
+        }
         const nix::Attr & a = (*v.attrs())[i];
         return state->state.symbols[a.name].c_str();
     }
@@ -594,7 +679,7 @@ nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * b
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_not_null(value);
-        nix::Symbol s = bb->builder.state.get().symbols.create(name);
+        nix::Symbol s = bb->builder.symbols.get().create(name);
         bb->builder.insert(s, &v);
     }
     NIXC_CATCH_ERRS

src/libexpr-c/nix_api_value.h

@@ -265,10 +265,25 @@ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value);
  */
 nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
 
-/** @brief Get an attr by name
+/** @brief Get the ix'th element of a list without forcing evaluation of the element
+ *
+ * Returns the list element without forcing its evaluation, allowing access to lazy values.
+ * The list value itself must already be evaluated.
  *
  * Owned by the GC. Use nix_gc_decref when you're done with the pointer
  * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect (must be an evaluated list)
+ * @param[in] state nix evaluator state
+ * @param[in] ix list element to get
+ * @return value, NULL in case of errors
+ */
+nix_value *
+nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
+
+/** @brief Get an attr by name
+ *
+ * Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @param[in] state nix evaluator state
  * @param[in] name attribute name
@@ -276,6 +291,21 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
  */
 nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
 
+/** @brief Get an attribute value by attribute name, without forcing evaluation of the attribute's value
+ *
+ * Returns the attribute value without forcing its evaluation, allowing access to lazy values.
+ * The attribute set value itself must already be evaluated.
+ *
+ * Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect (must be an evaluated attribute set)
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, NULL in case of errors
+ */
+nix_value *
+nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
+
 /** @brief Check if an attribute name exists on a value
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
@@ -285,11 +315,21 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
  */
 bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
 
-/** @brief Get an attribute by index in the sorted bindings
+/** @brief Get an attribute by index
  *
  * Also gives you the name.
  *
- * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * Attributes are returned in an unspecified order which is NOT suitable for
+ * reproducible operations. In Nix's domain, reproducibility is paramount. The caller
+ * is responsible for sorting the attributes or storing them in an ordered map to
+ * ensure deterministic behavior in your application.
+ *
+ * @note When Nix does sort attributes, which it does for virtually all intermediate
+ * operations and outputs, it uses byte-wise lexicographic order (equivalent to
+ * lexicographic order by Unicode scalar value for valid UTF-8). We recommend
+ * applying this same ordering for consistency.
+ *
+ * Use nix_gc_decref when you're done with the pointer
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @param[in] state nix evaluator state
@@ -297,12 +337,50 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS
  * @param[out] name will store a pointer to the attribute name
  * @return value, NULL in case of errors
  */
-nix_value * nix_get_attr_byidx(
+nix_value *
-    nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name);
+nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name);
 
-/** @brief Get an attribute name by index in the sorted bindings
+/** @brief Get an attribute by index, without forcing evaluation of the attribute's value
  *
- * Useful when you want the name but want to avoid evaluation.
+ * Also gives you the name.
+ *
+ * Returns the attribute value without forcing its evaluation, allowing access to lazy values.
+ * The attribute set value itself must already have been evaluated.
+ *
+ * Attributes are returned in an unspecified order which is NOT suitable for
+ * reproducible operations. In Nix's domain, reproducibility is paramount. The caller
+ * is responsible for sorting the attributes or storing them in an ordered map to
+ * ensure deterministic behavior in your application.
+ *
+ * @note When Nix does sort attributes, which it does for virtually all intermediate
+ * operations and outputs, it uses byte-wise lexicographic order (equivalent to
+ * lexicographic order by Unicode scalar value for valid UTF-8). We recommend
+ * applying this same ordering for consistency.
+ *
+ * Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect (must be an evaluated attribute set)
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @param[out] name will store a pointer to the attribute name
+ * @return value, NULL in case of errors
+ */
+nix_value * nix_get_attr_byidx_lazy(
+    nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name);
+
+/** @brief Get an attribute name by index
+ *
+ * Returns the attribute name without forcing evaluation of the attribute's value.
+ *
+ * Attributes are returned in an unspecified order which is NOT suitable for
+ * reproducible operations. In Nix's domain, reproducibility is paramount. The caller
+ * is responsible for sorting the attributes or storing them in an ordered map to
+ * ensure deterministic behavior in your application.
+ *
+ * @note When Nix does sort attributes, which it does for virtually all intermediate
+ * operations and outputs, it uses byte-wise lexicographic order (equivalent to
+ * lexicographic order by Unicode scalar value for valid UTF-8). We recommend
+ * applying this same ordering for consistency.
  *
  * Owned by the nix EvalState
  * @param[out] context Optional, stores error information
@@ -311,8 +389,7 @@ nix_value * nix_get_attr_byidx(
  * @param[in] i attribute index
  * @return name, NULL in case of errors
  */
-const char *
+const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i);
-nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i);
 
 /**@}*/
 /** @name Initializers

src/libexpr-test-support/meson.build

@@ -31,6 +31,7 @@ rapidcheck = dependency('rapidcheck')
 deps_public += rapidcheck
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'tests/value/context.cc',

src/libexpr-tests/main.cc

@@ -1,40 +1,15 @@
 #include <gtest/gtest.h>
-#include <cstdlib>
+
-#include "nix/store/globals.hh"
+#include "nix/store/tests/test-main.hh"
-#include "nix/util/logging.hh"
+#include "nix/util/config-global.hh"
 
 using namespace nix;
 
 int main(int argc, char ** argv)
 {
-    if (argc > 1 && std::string_view(argv[1]) == "__build-remote") {
+    auto res = testMainForBuidingPre(argc, argv);
-        printError("test-build-remote: not supported in libexpr unit tests");
+    if (res)
-        return 1;
+        return res;
-    }
-
-    // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook.
-    settings.buildHook = {};
-
-#ifdef __linux__ // should match the conditional around sandboxBuildDir declaration.
-
-    // When building and testing nix within the host's Nix sandbox, our store dir will be located in the host's
-    // sandboxBuildDir, e.g.: Host
-    //   storeDir = /nix/store
-    //   sandboxBuildDir = /build
-    // This process
-    //   storeDir = /build/foo/bar/store
-    //   sandboxBuildDir = /build
-    // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different
-    // sandboxBuildDir.
-    settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
-#endif
-
-#ifdef __APPLE__
-    // Avoid this error, when already running in a sandbox:
-    // sandbox-exec: sandbox_apply: Operation not permitted
-    settings.sandboxMode = smDisabled;
-    setEnv("_NIX_TEST_NO_SANDBOX", "1");
-#endif
 
     // For pipe operator tests in trivial.cc
     experimentalFeatureSettings.set("experimental-features", "pipe-operators");

src/libexpr-tests/meson.build

@@ -45,6 +45,7 @@ config_priv_h = configure_file(
 )
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'derived-path.cc',
@@ -82,7 +83,7 @@ this_exe = executable(
 test(
   meson.project_name(),
   this_exe,
-  env : {
+  env : asan_test_options_env + {
     '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data',
   },
   protocol : 'gtest',

src/libexpr-tests/nix_api_expr.cc

@@ -423,6 +423,55 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk)
     ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("badReturnThunk"));
 }
 
+static void primop_with_nix_err_key(
+    void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret)
+{
+    nix_set_err_msg(context, NIX_ERR_KEY, "Test error from primop");
+}
+
+TEST_F(nix_api_expr_test, nix_expr_primop_nix_err_key_conversion)
+{
+    // Test that NIX_ERR_KEY from a custom primop gets converted to a generic EvalError
+    //
+    // RATIONALE: NIX_ERR_KEY must not be propagated from custom primops because it would
+    // create semantic confusion. NIX_ERR_KEY indicates missing keys/indices in C API functions
+    // (like nix_get_attr_byname, nix_get_list_byidx). If custom primops could return NIX_ERR_KEY,
+    // an evaluation error would be indistinguishable from an actual missing attribute.
+    //
+    // For example, if nix_get_attr_byname returned NIX_ERR_KEY when the attribute is present
+    // but the value evaluation fails, callers expecting NIX_ERR_KEY to mean "missing attribute"
+    // would incorrectly handle evaluation failures as missing attributes. In places where
+    // missing attributes are tolerated (like optional attributes), this would cause the
+    // program to continue after swallowing the error, leading to silent failures.
+    PrimOp * primop = nix_alloc_primop(
+        ctx, primop_with_nix_err_key, 1, "testErrorPrimop", nullptr, "a test primop that sets NIX_ERR_KEY", nullptr);
+    assert_ctx_ok();
+    nix_value * primopValue = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    nix_init_primop(ctx, primopValue, primop);
+    assert_ctx_ok();
+
+    nix_value * arg = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    nix_init_int(ctx, arg, 42);
+    assert_ctx_ok();
+
+    nix_value * result = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    nix_value_call(ctx, state, primopValue, arg, result);
+
+    // Verify that NIX_ERR_KEY gets converted to NIX_ERR_NIX_ERROR (generic evaluation error)
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("Error from custom function"));
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("Test error from primop"));
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("testErrorPrimop"));
+
+    // Clean up
+    nix_gc_decref(ctx, primopValue);
+    nix_gc_decref(ctx, arg);
+    nix_gc_decref(ctx, result);
+}
+
 TEST_F(nix_api_expr_test, nix_value_call_multi_no_args)
 {
     nix_value * n = nix_alloc_value(ctx, state);
@@ -437,4 +486,31 @@ TEST_F(nix_api_expr_test, nix_value_call_multi_no_args)
     assert_ctx_ok();
     ASSERT_EQ(3, rInt);
 }
+
+TEST_F(nix_api_expr_test, nix_expr_attrset_update)
+{
+    nix_expr_eval_from_string(ctx, state, "{ a = 0; b = 2; } // { a = 1; b = 3; } // { a = 2; }", ".", value);
+    assert_ctx_ok();
+
+    ASSERT_EQ(nix_get_attrs_size(ctx, value), 2);
+    assert_ctx_ok();
+    std::array<std::pair<std::string_view, nix_value *>, 2> values;
+    for (unsigned int i = 0; i < 2; ++i) {
+        const char * name;
+        values[i].second = nix_get_attr_byidx(ctx, value, state, i, &name);
+        assert_ctx_ok();
+        values[i].first = name;
+    }
+    std::sort(values.begin(), values.end(), [](const auto & lhs, const auto & rhs) { return lhs.first < rhs.first; });
+
+    nix_value * a = values[0].second;
+    ASSERT_EQ("a", values[0].first);
+    ASSERT_EQ(nix_get_int(ctx, a), 2);
+    assert_ctx_ok();
+    nix_value * b = values[1].second;
+    ASSERT_EQ("b", values[1].first);
+    ASSERT_EQ(nix_get_int(ctx, b), 3);
+    assert_ctx_ok();
+}
+
 } // namespace nixC

src/libexpr-tests/nix_api_value.cc

@@ -162,6 +162,114 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list)
     nix_gc_decref(ctx, intValue);
 }
 
+TEST_F(nix_api_expr_test, nix_get_list_byidx_large_indices)
+{
+    // Create a small list to test extremely large out-of-bounds access
+    ListBuilder * builder = nix_make_list_builder(ctx, state, 2);
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 42);
+    nix_list_builder_insert(ctx, builder, 0, intValue);
+    nix_list_builder_insert(ctx, builder, 1, intValue);
+    nix_make_list(ctx, builder, value);
+    nix_list_builder_free(builder);
+
+    // Test extremely large indices that would definitely crash without bounds checking
+    ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 1000000));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, UINT_MAX / 2));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, UINT_MAX / 2 + 1000000));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+
+    // Clean up
+    nix_gc_decref(ctx, intValue);
+}
+
+TEST_F(nix_api_expr_test, nix_get_list_byidx_lazy)
+{
+    // Create a list with a throwing lazy element, an already-evaluated int, and a lazy function call
+
+    // 1. Throwing lazy element - create a function application thunk that will throw when forced
+    nix_value * throwingFn = nix_alloc_value(ctx, state);
+    nix_value * throwingValue = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(
+        ctx,
+        state,
+        R"(
+        _: throw "This should not be evaluated by the lazy accessor"
+    )",
+        "<test>",
+        throwingFn);
+    assert_ctx_ok();
+
+    nix_init_apply(ctx, throwingValue, throwingFn, throwingFn);
+    assert_ctx_ok();
+
+    // 2. Already evaluated int (not lazy)
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 42);
+    assert_ctx_ok();
+
+    // 3. Lazy function application that would compute increment 5 = 6
+    nix_value * lazyApply = nix_alloc_value(ctx, state);
+    nix_value * incrementFn = nix_alloc_value(ctx, state);
+    nix_value * argFive = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(ctx, state, "x: x + 1", "<test>", incrementFn);
+    assert_ctx_ok();
+    nix_init_int(ctx, argFive, 5);
+
+    // Create a lazy application: (x: x + 1) 5
+    nix_init_apply(ctx, lazyApply, incrementFn, argFive);
+    assert_ctx_ok();
+
+    ListBuilder * builder = nix_make_list_builder(ctx, state, 3);
+    nix_list_builder_insert(ctx, builder, 0, throwingValue);
+    nix_list_builder_insert(ctx, builder, 1, intValue);
+    nix_list_builder_insert(ctx, builder, 2, lazyApply);
+    nix_make_list(ctx, builder, value);
+    nix_list_builder_free(builder);
+
+    // Test 1: Lazy accessor should return the throwing element without forcing evaluation
+    nix_value * lazyThrowingElement = nix_get_list_byidx_lazy(ctx, value, state, 0);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lazyThrowingElement);
+
+    // Verify the element is still lazy by checking that forcing it throws
+    nix_value_force(ctx, state, lazyThrowingElement);
+    assert_ctx_err();
+    ASSERT_THAT(
+        nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor"));
+
+    // Test 2: Lazy accessor should return the already-evaluated int
+    nix_value * intElement = nix_get_list_byidx_lazy(ctx, value, state, 1);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, intElement);
+    ASSERT_EQ(42, nix_get_int(ctx, intElement));
+
+    // Test 3: Lazy accessor should return the lazy function application without forcing
+    nix_value * lazyFunctionElement = nix_get_list_byidx_lazy(ctx, value, state, 2);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lazyFunctionElement);
+
+    // Force the lazy function application - should compute 5 + 1 = 6
+    nix_value_force(ctx, state, lazyFunctionElement);
+    assert_ctx_ok();
+    ASSERT_EQ(6, nix_get_int(ctx, lazyFunctionElement));
+
+    // Clean up
+    nix_gc_decref(ctx, throwingFn);
+    nix_gc_decref(ctx, throwingValue);
+    nix_gc_decref(ctx, intValue);
+    nix_gc_decref(ctx, lazyApply);
+    nix_gc_decref(ctx, incrementFn);
+    nix_gc_decref(ctx, argFive);
+    nix_gc_decref(ctx, lazyThrowingElement);
+    nix_gc_decref(ctx, intElement);
+    nix_gc_decref(ctx, lazyFunctionElement);
+}
+
 TEST_F(nix_api_expr_test, nix_build_and_init_attr_invalid)
 {
     ASSERT_EQ(nullptr, nix_get_attr_byname(ctx, nullptr, state, 0));
@@ -244,6 +352,225 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr)
     free(out_name);
 }
 
+TEST_F(nix_api_expr_test, nix_get_attr_byidx_large_indices)
+{
+    // Create a small attribute set to test extremely large out-of-bounds access
+    const char ** out_name = (const char **) malloc(sizeof(char *));
+    BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 2);
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 42);
+    nix_bindings_builder_insert(ctx, builder, "test", intValue);
+    nix_make_attrs(ctx, value, builder);
+    nix_bindings_builder_free(builder);
+
+    // Test extremely large indices that would definitely crash without bounds checking
+    ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, 1000000, out_name));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, UINT_MAX / 2, out_name));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, UINT_MAX / 2 + 1000000, out_name));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+
+    // Test nix_get_attr_name_byidx with large indices too
+    ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, 1000000));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, UINT_MAX / 2));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+    ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, UINT_MAX / 2 + 1000000));
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+
+    // Clean up
+    nix_gc_decref(ctx, intValue);
+    free(out_name);
+}
+
+TEST_F(nix_api_expr_test, nix_get_attr_byname_lazy)
+{
+    // Create an attribute set with a throwing lazy attribute, an already-evaluated int, and a lazy function call
+
+    // 1. Throwing lazy element - create a function application thunk that will throw when forced
+    nix_value * throwingFn = nix_alloc_value(ctx, state);
+    nix_value * throwingValue = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(
+        ctx,
+        state,
+        R"(
+        _: throw "This should not be evaluated by the lazy accessor"
+    )",
+        "<test>",
+        throwingFn);
+    assert_ctx_ok();
+
+    nix_init_apply(ctx, throwingValue, throwingFn, throwingFn);
+    assert_ctx_ok();
+
+    // 2. Already evaluated int (not lazy)
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 42);
+    assert_ctx_ok();
+
+    // 3. Lazy function application that would compute increment 7 = 8
+    nix_value * lazyApply = nix_alloc_value(ctx, state);
+    nix_value * incrementFn = nix_alloc_value(ctx, state);
+    nix_value * argSeven = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(ctx, state, "x: x + 1", "<test>", incrementFn);
+    assert_ctx_ok();
+    nix_init_int(ctx, argSeven, 7);
+
+    // Create a lazy application: (x: x + 1) 7
+    nix_init_apply(ctx, lazyApply, incrementFn, argSeven);
+    assert_ctx_ok();
+
+    BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 3);
+    nix_bindings_builder_insert(ctx, builder, "throwing", throwingValue);
+    nix_bindings_builder_insert(ctx, builder, "normal", intValue);
+    nix_bindings_builder_insert(ctx, builder, "lazy", lazyApply);
+    nix_make_attrs(ctx, value, builder);
+    nix_bindings_builder_free(builder);
+
+    // Test 1: Lazy accessor should return the throwing attribute without forcing evaluation
+    nix_value * lazyThrowingAttr = nix_get_attr_byname_lazy(ctx, value, state, "throwing");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lazyThrowingAttr);
+
+    // Verify the attribute is still lazy by checking that forcing it throws
+    nix_value_force(ctx, state, lazyThrowingAttr);
+    assert_ctx_err();
+    ASSERT_THAT(
+        nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor"));
+
+    // Test 2: Lazy accessor should return the already-evaluated int
+    nix_value * intAttr = nix_get_attr_byname_lazy(ctx, value, state, "normal");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, intAttr);
+    ASSERT_EQ(42, nix_get_int(ctx, intAttr));
+
+    // Test 3: Lazy accessor should return the lazy function application without forcing
+    nix_value * lazyFunctionAttr = nix_get_attr_byname_lazy(ctx, value, state, "lazy");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lazyFunctionAttr);
+
+    // Force the lazy function application - should compute 7 + 1 = 8
+    nix_value_force(ctx, state, lazyFunctionAttr);
+    assert_ctx_ok();
+    ASSERT_EQ(8, nix_get_int(ctx, lazyFunctionAttr));
+
+    // Test 4: Missing attribute should return NULL with NIX_ERR_KEY
+    nix_value * missingAttr = nix_get_attr_byname_lazy(ctx, value, state, "nonexistent");
+    ASSERT_EQ(nullptr, missingAttr);
+    ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
+
+    // Clean up
+    nix_gc_decref(ctx, throwingFn);
+    nix_gc_decref(ctx, throwingValue);
+    nix_gc_decref(ctx, intValue);
+    nix_gc_decref(ctx, lazyApply);
+    nix_gc_decref(ctx, incrementFn);
+    nix_gc_decref(ctx, argSeven);
+    nix_gc_decref(ctx, lazyThrowingAttr);
+    nix_gc_decref(ctx, intAttr);
+    nix_gc_decref(ctx, lazyFunctionAttr);
+}
+
+TEST_F(nix_api_expr_test, nix_get_attr_byidx_lazy)
+{
+    // Create an attribute set with a throwing lazy attribute, an already-evaluated int, and a lazy function call
+
+    // 1. Throwing lazy element - create a function application thunk that will throw when forced
+    nix_value * throwingFn = nix_alloc_value(ctx, state);
+    nix_value * throwingValue = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(
+        ctx,
+        state,
+        R"(
+        _: throw "This should not be evaluated by the lazy accessor"
+    )",
+        "<test>",
+        throwingFn);
+    assert_ctx_ok();
+
+    nix_init_apply(ctx, throwingValue, throwingFn, throwingFn);
+    assert_ctx_ok();
+
+    // 2. Already evaluated int (not lazy)
+    nix_value * intValue = nix_alloc_value(ctx, state);
+    nix_init_int(ctx, intValue, 99);
+    assert_ctx_ok();
+
+    // 3. Lazy function application that would compute increment 10 = 11
+    nix_value * lazyApply = nix_alloc_value(ctx, state);
+    nix_value * incrementFn = nix_alloc_value(ctx, state);
+    nix_value * argTen = nix_alloc_value(ctx, state);
+
+    nix_expr_eval_from_string(ctx, state, "x: x + 1", "<test>", incrementFn);
+    assert_ctx_ok();
+    nix_init_int(ctx, argTen, 10);
+
+    // Create a lazy application: (x: x + 1) 10
+    nix_init_apply(ctx, lazyApply, incrementFn, argTen);
+    assert_ctx_ok();
+
+    BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 3);
+    nix_bindings_builder_insert(ctx, builder, "a_throwing", throwingValue);
+    nix_bindings_builder_insert(ctx, builder, "b_normal", intValue);
+    nix_bindings_builder_insert(ctx, builder, "c_lazy", lazyApply);
+    nix_make_attrs(ctx, value, builder);
+    nix_bindings_builder_free(builder);
+
+    // Proper usage: first get the size and gather all attributes into a map
+    unsigned int attrCount = nix_get_attrs_size(ctx, value);
+    assert_ctx_ok();
+    ASSERT_EQ(3u, attrCount);
+
+    // Gather all attributes into a map (proper contract usage)
+    std::map<std::string, nix_value *> attrMap;
+    const char * name;
+
+    for (unsigned int i = 0; i < attrCount; i++) {
+        nix_value * attr = nix_get_attr_byidx_lazy(ctx, value, state, i, &name);
+        assert_ctx_ok();
+        ASSERT_NE(nullptr, attr);
+        attrMap[std::string(name)] = attr;
+    }
+
+    // Now test the gathered attributes
+    ASSERT_EQ(3u, attrMap.size());
+    ASSERT_TRUE(attrMap.count("a_throwing"));
+    ASSERT_TRUE(attrMap.count("b_normal"));
+    ASSERT_TRUE(attrMap.count("c_lazy"));
+
+    // Test 1: Throwing attribute should be lazy
+    nix_value * throwingAttr = attrMap["a_throwing"];
+    nix_value_force(ctx, state, throwingAttr);
+    assert_ctx_err();
+    ASSERT_THAT(
+        nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor"));
+
+    // Test 2: Normal attribute should be already evaluated
+    nix_value * normalAttr = attrMap["b_normal"];
+    ASSERT_EQ(99, nix_get_int(ctx, normalAttr));
+
+    // Test 3: Lazy function should compute when forced
+    nix_value * lazyAttr = attrMap["c_lazy"];
+    nix_value_force(ctx, state, lazyAttr);
+    assert_ctx_ok();
+    ASSERT_EQ(11, nix_get_int(ctx, lazyAttr));
+
+    // Clean up
+    nix_gc_decref(ctx, throwingFn);
+    nix_gc_decref(ctx, throwingValue);
+    nix_gc_decref(ctx, intValue);
+    nix_gc_decref(ctx, lazyApply);
+    nix_gc_decref(ctx, incrementFn);
+    nix_gc_decref(ctx, argTen);
+    for (auto & pair : attrMap) {
+        nix_gc_decref(ctx, pair.second);
+    }
+}
+
 TEST_F(nix_api_expr_test, nix_value_init)
 {
     // Setup

src/libexpr-tests/package.nix

@@ -62,6 +62,7 @@ mkMesonExecutable (finalAttrs: {
               mkdir -p "$HOME"
             ''
             + ''
+              export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0
               export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
               ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
               touch $out

src/libexpr-tests/primops.cc

@@ -642,7 +642,7 @@ class ToStringPrimOpTest : public PrimOpTest,
 
 TEST_P(ToStringPrimOpTest, toString)
 {
-    const auto [input, output] = GetParam();
+    const auto & [input, output] = GetParam();
     auto v = eval(input);
     ASSERT_THAT(v, IsStringEq(output));
 }
@@ -798,7 +798,7 @@ class CompareVersionsPrimOpTest : public PrimOpTest,
 
 TEST_P(CompareVersionsPrimOpTest, compareVersions)
 {
-    auto [expression, expectation] = GetParam();
+    const auto & [expression, expectation] = GetParam();
     auto v = eval(expression);
     ASSERT_THAT(v, IsIntEq(expectation));
 }
@@ -834,7 +834,7 @@ class ParseDrvNamePrimOpTest
 
 TEST_P(ParseDrvNamePrimOpTest, parseDrvName)
 {
-    auto [input, expectedName, expectedVersion] = GetParam();
+    const auto & [input, expectedName, expectedVersion] = GetParam();
     const auto expr = fmt("builtins.parseDrvName \"%1%\"", input);
     auto v = eval(expr);
     ASSERT_THAT(v, IsAttrsOfSize(2));

src/libexpr/attr-set.cc

@@ -10,32 +10,32 @@ Bindings Bindings::emptyBindings;
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
-Bindings * EvalState::allocBindings(size_t capacity)
+Bindings * EvalMemory::allocBindings(size_t capacity)
 {
     if (capacity == 0)
         return &Bindings::emptyBindings;
-    if (capacity > std::numeric_limits<Bindings::size_t>::max())
+    if (capacity > std::numeric_limits<Bindings::size_type>::max())
         throw Error("attribute set of size %d is too big", capacity);
-    nrAttrsets++;
+    stats.nrAttrsets++;
-    nrAttrsInAttrsets += capacity;
+    stats.nrAttrsInAttrsets += capacity;
     return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings();
 }
 
 Value & BindingsBuilder::alloc(Symbol name, PosIdx pos)
 {
-    auto value = state.get().allocValue();
+    auto value = mem.get().allocValue();
     bindings->push_back(Attr(name, value, pos));
     return *value;
 }
 
 Value & BindingsBuilder::alloc(std::string_view name, PosIdx pos)
 {
-    return alloc(state.get().symbols.create(name), pos);
+    return alloc(symbols.get().create(name), pos);
 }
 
 void Bindings::sort()
 {
-    std::sort(attrs, attrs + size_);
+    std::sort(attrs, attrs + numAttrs);
 }
 
 Value & Value::mkAttrs(BindingsBuilder & bindings)

src/libexpr/eval.cc

@@ -17,6 +17,7 @@
 #include "nix/expr/print.hh"
 #include "nix/fetchers/filtering-source-accessor.hh"
 #include "nix/util/memory-source-accessor.hh"
+#include "nix/util/mounted-source-accessor.hh"
 #include "nix/expr/gc-small-vector.hh"
 #include "nix/util/url.hh"
 #include "nix/fetchers/fetch-to-store.hh"
@@ -193,6 +194,15 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
 
 static constexpr size_t BASE_ENV_SIZE = 128;
 
+EvalMemory::EvalMemory()
+#if NIX_USE_BOEHMGC
+    : valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
+    , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
+#endif
+{
+    assertGCInitialized();
+}
+
 EvalState::EvalState(
     const LookupPath & lookupPathFromArguments,
     ref<Store> store,
@@ -225,22 +235,25 @@ EvalState::EvalState(
            */
           {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
       }))
-    , rootFS(({
+    , rootFS([&] {
+        auto accessor = [&]() -> decltype(rootFS) {
             /* In pure eval mode, we provide a filesystem that only
-           contains the Nix store.
+               contains the Nix store. */
+            if (settings.pureEval)
+                return storeFS;
 
-           If we have a chroot store and pure eval is not enabled,
+            /* If we have a chroot store and pure eval is not enabled,
                use a union accessor to make the chroot store available
-           at its logical location while still having the
+               at its logical location while still having the underlying
-           underlying directory available. This is necessary for
+               directory available. This is necessary for instance if
-           instance if we're evaluating a file from the physical
+               we're evaluating a file from the physical /nix/store
-           /nix/store while using a chroot store. */
+               while using a chroot store. */
-        auto accessor = getFSSourceAccessor();
-
             auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
-        if (settings.pureEval || store->storeDir != realStoreDir) {
+            if (store->storeDir != realStoreDir)
-            accessor = settings.pureEval ? storeFS : makeUnionSourceAccessor({accessor, storeFS});
+                return makeUnionSourceAccessor({getFSSourceAccessor(), storeFS});
-        }
+
+            return getFSSourceAccessor();
+        }();
 
         /* Apply access control if needed. */
         if (settings.restrictEval || settings.pureEval)
@@ -251,8 +264,8 @@ EvalState::EvalState(
                     throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
                 });
 
-        accessor;
+        return accessor;
-    }))
+    }())
     , corepkgsFS(make_ref<MemorySourceAccessor>())
     , internalFS(make_ref<MemorySourceAccessor>())
     , derivationInternal{corepkgsFS->addFile(
@@ -270,12 +283,10 @@ EvalState::EvalState(
     , fileEvalCache(make_ref<decltype(fileEvalCache)::element_type>())
     , regexCache(makeRegexCache())
 #if NIX_USE_BOEHMGC
-    , valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
+    , baseEnvP(std::allocate_shared<Env *>(traceable_allocator<Env *>(), &mem.allocEnv(BASE_ENV_SIZE)))
-    , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
-    , baseEnvP(std::allocate_shared<Env *>(traceable_allocator<Env *>(), &allocEnv(BASE_ENV_SIZE)))
     , baseEnv(**baseEnvP)
 #else
-    , baseEnv(allocEnv(BASE_ENV_SIZE))
+    , baseEnv(mem.allocEnv(BASE_ENV_SIZE))
 #endif
     , staticBaseEnv{std::make_shared<StaticEnv>(nullptr, nullptr)}
 {
@@ -284,9 +295,8 @@ EvalState::EvalState(
 
     countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
 
-    assertGCInitialized();
-
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
+    static_assert(sizeof(Counter) == 64, "counters must be 64 bytes");
 
     /* Construct the Nix expression search path. */
     assert(lookupPath.elements.empty());
@@ -333,7 +343,7 @@ EvalState::EvalState(
 
 EvalState::~EvalState() {}
 
-void EvalState::allowPath(const Path & path)
+void EvalState::allowPathLegacy(const Path & path)
 {
     if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
         rootFS2->allowPrefix(CanonPath(path));
@@ -880,11 +890,10 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
     }
 }
 
-ListBuilder::ListBuilder(EvalState & state, size_t size)
+ListBuilder::ListBuilder(size_t size)
     : size(size)
     , elems(size <= 2 ? inlineElems : (Value **) allocBytes(size * sizeof(Value *)))
 {
-    state.nrListElems += size;
 }
 
 Value * EvalState::getBool(bool b)
@@ -892,7 +901,7 @@ Value * EvalState::getBool(bool b)
     return b ? &Value::vTrue : &Value::vFalse;
 }
 
-unsigned long nrThunks = 0;
+static Counter nrThunks;
 
 static inline void mkThunk(Value & v, Env & env, Expr * expr)
 {
@@ -983,10 +992,6 @@ void EvalState::mkSingleDerivedPathString(const SingleDerivedPath & p, Value & v
         });
 }
 
-/* Create a thunk for the delayed computation of the given expression
-   in the given environment.  But if the expression is a variable,
-   then look it up right away.  This significantly reduces the number
-   of thunks allocated. */
 Value * Expr::maybeThunk(EvalState & state, Env & env)
 {
     Value * v = state.allocValue();
@@ -1035,9 +1040,10 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
  * from a thunk, ensuring that every file is parsed/evaluated only
  * once (via the thunk stored in `EvalState::fileEvalCache`).
  */
-struct ExprParseFile : Expr
+struct ExprParseFile : Expr, gc
 {
-    SourcePath & path;
+    // FIXME: make this a reference (see below).
+    SourcePath path;
     bool mustBeTrivial;
 
     ExprParseFile(SourcePath & path, bool mustBeTrivial)
@@ -1088,14 +1094,18 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
     }
 
     Value * vExpr;
-    ExprParseFile expr{*resolvedPath, mustBeTrivial};
+    // FIXME: put ExprParseFile on the stack instead of the heap once
+    // https://github.com/NixOS/nix/pull/13930 is merged. That will ensure
+    // the post-condition that `expr` is unreachable after
+    // `forceValue()` returns.
+    auto expr = new ExprParseFile{*resolvedPath, mustBeTrivial};
 
     fileEvalCache->try_emplace_and_cvisit(
         *resolvedPath,
         nullptr,
         [&](auto & i) {
             vExpr = allocValue();
-            vExpr->mkThunk(&baseEnv, &expr);
+            vExpr->mkThunk(&baseEnv, expr);
             i.second = vExpr;
         },
         [&](auto & i) { vExpr = i.second; });
@@ -1177,7 +1187,7 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v)
 
 Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
 {
-    Env & inheritEnv = state.allocEnv(inheritFromExprs->size());
+    Env & inheritEnv = state.mem.allocEnv(inheritFromExprs->size());
     inheritEnv.up = &up;
 
     Displacement displ = 0;
@@ -1196,7 +1206,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
     if (recursive) {
         /* Create a new environment that contains the attributes in
            this `rec'. */
-        Env & env2(state.allocEnv(attrs.size()));
+        Env & env2(state.mem.allocEnv(attrs.size()));
         env2.up = &env;
         dynamicEnv = &env2;
         Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
@@ -1288,7 +1298,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 {
     /* Create a new environment that contains the attributes in this
        `let'. */
-    Env & env2(state.allocEnv(attrs->attrs.size()));
+    Env & env2(state.mem.allocEnv(attrs->attrs.size()));
     env2.up = &env;
 
     Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
@@ -1494,7 +1504,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
             ExprLambda & lambda(*vCur.lambda().fun);
 
             auto size = (!lambda.arg ? 0 : 1) + (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
-            Env & env2(allocEnv(size));
+            Env & env2(mem.allocEnv(size));
             env2.up = vCur.lambda().env;
 
             Displacement displ = 0;
@@ -1783,7 +1793,7 @@ https://nix.dev/manual/nix/stable/language/syntax.html#functions.)",
 
 void ExprWith::eval(EvalState & state, Env & env, Value & v)
 {
-    Env & env2(state.allocEnv(1));
+    Env & env2(state.mem.allocEnv(1));
     env2.up = &env;
     env2.values[0] = attrs->maybeThunk(state, env);
 
@@ -1865,51 +1875,113 @@ void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
         || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator"));
 }
 
-void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
+void ExprOpUpdate::eval(EvalState & state, Value & v, Value & v1, Value & v2)
 {
-    Value v1, v2;
-    state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator");
-    state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator");
-
     state.nrOpUpdates++;
 
-    if (v1.attrs()->size() == 0) {
+    const Bindings & bindings1 = *v1.attrs();
+    if (bindings1.empty()) {
         v = v2;
         return;
     }
-    if (v2.attrs()->size() == 0) {
+
+    const Bindings & bindings2 = *v2.attrs();
+    if (bindings2.empty()) {
         v = v1;
         return;
     }
 
-    auto attrs = state.buildBindings(v1.attrs()->size() + v2.attrs()->size());
+    /* Simple heuristic for determining whether attrs2 should be "layered" on top of
+       attrs1 instead of copying to a new Bindings. */
+    bool shouldLayer = [&]() -> bool {
+        if (bindings1.isLayerListFull())
+            return false;
+
+        if (bindings2.size() > state.settings.bindingsUpdateLayerRhsSizeThreshold)
+            return false;
+
+        return true;
+    }();
+
+    if (shouldLayer) {
+        auto attrs = state.buildBindings(bindings2.size());
+        attrs.layerOnTopOf(bindings1);
+
+        std::ranges::copy(bindings2, std::back_inserter(attrs));
+        v.mkAttrs(attrs.alreadySorted());
+
+        state.nrOpUpdateValuesCopied += bindings2.size();
+        return;
+    }
+
+    auto attrs = state.buildBindings(bindings1.size() + bindings2.size());
 
     /* Merge the sets, preferring values from the second set.  Make
        sure to keep the resulting vector in sorted order. */
-    auto i = v1.attrs()->begin();
+    auto i = bindings1.begin();
-    auto j = v2.attrs()->begin();
+    auto j = bindings2.begin();
 
-    while (i != v1.attrs()->end() && j != v2.attrs()->end()) {
+    while (i != bindings1.end() && j != bindings2.end()) {
         if (i->name == j->name) {
             attrs.insert(*j);
             ++i;
             ++j;
-        } else if (i->name < j->name)
+        } else if (i->name < j->name) {
-            attrs.insert(*i++);
+            attrs.insert(*i);
-        else
+            ++i;
-            attrs.insert(*j++);
+        } else {
+            attrs.insert(*j);
+            ++j;
+        }
     }
 
-    while (i != v1.attrs()->end())
+    while (i != bindings1.end()) {
-        attrs.insert(*i++);
+        attrs.insert(*i);
-    while (j != v2.attrs()->end())
+        ++i;
-        attrs.insert(*j++);
+    }
+
+    while (j != bindings2.end()) {
+        attrs.insert(*j);
+        ++j;
+    }
 
     v.mkAttrs(attrs.alreadySorted());
 
     state.nrOpUpdateValuesCopied += v.attrs()->size();
 }
 
+void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
+{
+    UpdateQueue q;
+    evalForUpdate(state, env, q);
+
+    v.mkAttrs(&Bindings::emptyBindings);
+    for (auto & rhs : std::views::reverse(q)) {
+        /* Remember that queue is sorted rightmost attrset first. */
+        eval(state, /*v=*/v, /*v1=*/v, /*v2=*/rhs);
+    }
+}
+
+void Expr::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx)
+{
+    Value v;
+    state.evalAttrs(env, this, v, getPos(), errorCtx);
+    q.push_back(v);
+}
+
+void ExprOpUpdate::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q)
+{
+    /* Output rightmost attrset first to the merge queue as the one
+       with the most priority. */
+    e2->evalForUpdate(state, env, q, "in the right operand of the update (//) operator");
+    e1->evalForUpdate(state, env, q, "in the left operand of the update (//) operator");
+}
+
+void ExprOpUpdate::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx)
+{
+    evalForUpdate(state, env, q);
+}
+
 void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
 {
     Value v1;
@@ -2828,11 +2900,11 @@ bool EvalState::fullGC()
 #endif
 }
 
+bool Counter::enabled = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
+
 void EvalState::maybePrintStats()
 {
-    bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
+    if (Counter::enabled) {
-
-    if (showStats) {
         // Make the final heap size more deterministic.
 #if NIX_USE_BOEHMGC
         if (!fullGC()) {
@@ -2848,10 +2920,12 @@ void EvalState::printStatistics()
     std::chrono::microseconds cpuTimeDuration = getCpuUserTime();
     float cpuTime = std::chrono::duration_cast<std::chrono::duration<float>>(cpuTimeDuration).count();
 
-    uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *);
+    auto & memstats = mem.getStats();
-    uint64_t bLists = nrListElems * sizeof(Value *);
+
-    uint64_t bValues = nrValues * sizeof(Value);
+    uint64_t bEnvs = memstats.nrEnvs * sizeof(Env) + memstats.nrValuesInEnvs * sizeof(Value *);
-    uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
+    uint64_t bLists = memstats.nrListElems * sizeof(Value *);
+    uint64_t bValues = memstats.nrValues * sizeof(Value);
+    uint64_t bAttrsets = memstats.nrAttrsets * sizeof(Bindings) + memstats.nrAttrsInAttrsets * sizeof(Attr);
 
 #if NIX_USE_BOEHMGC
     GC_word heapSize, totalBytes;
@@ -2877,18 +2951,18 @@ void EvalState::printStatistics()
 #endif
     };
     topObj["envs"] = {
-        {"number", nrEnvs},
+        {"number", memstats.nrEnvs.load()},
-        {"elements", nrValuesInEnvs},
+        {"elements", memstats.nrValuesInEnvs.load()},
         {"bytes", bEnvs},
     };
-    topObj["nrExprs"] = Expr::nrExprs;
+    topObj["nrExprs"] = Expr::nrExprs.load();
     topObj["list"] = {
-        {"elements", nrListElems},
+        {"elements", memstats.nrListElems.load()},
         {"bytes", bLists},
-        {"concats", nrListConcats},
+        {"concats", nrListConcats.load()},
     };
     topObj["values"] = {
-        {"number", nrValues},
+        {"number", memstats.nrValues.load()},
         {"bytes", bValues},
     };
     topObj["symbols"] = {
@@ -2896,9 +2970,9 @@ void EvalState::printStatistics()
         {"bytes", symbols.totalSize()},
     };
     topObj["sets"] = {
-        {"number", nrAttrsets},
+        {"number", memstats.nrAttrsets.load()},
         {"bytes", bAttrsets},
-        {"elements", nrAttrsInAttrsets},
+        {"elements", memstats.nrAttrsInAttrsets.load()},
     };
     topObj["sizes"] = {
         {"Env", sizeof(Env)},
@@ -2906,13 +2980,13 @@ void EvalState::printStatistics()
         {"Bindings", sizeof(Bindings)},
         {"Attr", sizeof(Attr)},
     };
-    topObj["nrOpUpdates"] = nrOpUpdates;
+    topObj["nrOpUpdates"] = nrOpUpdates.load();
-    topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied;
+    topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied.load();
-    topObj["nrThunks"] = nrThunks;
+    topObj["nrThunks"] = nrThunks.load();
-    topObj["nrAvoided"] = nrAvoided;
+    topObj["nrAvoided"] = nrAvoided.load();
-    topObj["nrLookups"] = nrLookups;
+    topObj["nrLookups"] = nrLookups.load();
-    topObj["nrPrimOpCalls"] = nrPrimOpCalls;
+    topObj["nrPrimOpCalls"] = nrPrimOpCalls.load();
-    topObj["nrFunctionCalls"] = nrFunctionCalls;
+    topObj["nrFunctionCalls"] = nrFunctionCalls.load();
 #if NIX_USE_BOEHMGC
     topObj["gc"] = {
         {"heapSize", heapSize},
@@ -3113,7 +3187,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
 
         /* Allow access to paths in the search path. */
         if (initAccessControl) {
-            allowPath(path.path.abs());
+            allowPathLegacy(path.path.abs());
             if (store->isInStore(path.path.abs())) {
                 try {
                     allowClosure(store->toStorePath(path.path.abs()).first);
@@ -3143,7 +3217,8 @@ Expr * EvalState::parse(
         docComments = &it->second;
     }
 
-    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS);
+    auto result = parseExprFromBuf(
+        text, length, origin, basePath, mem.exprs.alloc, symbols, settings, positions, *docComments, rootFS);
 
     result->bindVars(*this, staticEnv);
 

src/libexpr/include/nix/expr/attr-set.hh

@@ -4,13 +4,16 @@
 #include "nix/expr/nixexpr.hh"
 #include "nix/expr/symbol-table.hh"
 
+#include <boost/container/static_vector.hpp>
+
 #include <algorithm>
 #include <functional>
-#include <concepts>
+#include <ranges>
+#include <optional>
 
 namespace nix {
 
-class EvalState;
+class EvalMemory;
 struct Value;
 
 /**
@@ -48,11 +51,18 @@ static_assert(
  * by its size and its capacity, the capacity being the number of Attr
  * elements allocated after this structure, while the size corresponds to
  * the number of elements already inserted in this structure.
+ *
+ * Bindings can be efficiently `//`-composed into an intrusive linked list of "layers"
+ * that saves on copies and allocations. Each lookup (@see Bindings::get) traverses
+ * this linked list until a matching attribute is found (thus overlays earlier in
+ * the list take precedence). For iteration over the whole Bindings, an on-the-fly
+ * k-way merge is performed by Bindings::iterator class.
  */
 class Bindings
 {
 public:
-    typedef uint32_t size_t;
+    using size_type = uint32_t;
+
     PosIdx pos;
 
     /**
@@ -62,7 +72,32 @@ public:
     static Bindings emptyBindings;
 
 private:
-    size_t size_ = 0;
+    /**
+     * Number of attributes in the attrs FAM (Flexible Array Member).
+     */
+    size_type numAttrs = 0;
+
+    /**
+     * Number of attributes with unique names in the layer chain.
+     *
+     * This is the *real* user-facing size of bindings, whereas @ref numAttrs is
+     * an implementation detail of the data structure.
+     */
+    size_type numAttrsInChain = 0;
+
+    /**
+     * Length of the layers list.
+     */
+    uint32_t numLayers = 1;
+
+    /**
+     * Bindings that this attrset is "layered" on top of.
+     */
+    const Bindings * baseLayer = nullptr;
+
+    /**
+     * Flexible array member of attributes.
+     */
     Attr attrs[0];
 
     Bindings() = default;
@@ -71,15 +106,22 @@ private:
     Bindings & operator=(const Bindings &) = delete;
     Bindings & operator=(Bindings &&) = delete;
 
+    friend class BindingsBuilder;
+
+    /**
+     * Maximum length of the Bindings layer chains.
+     */
+    static constexpr unsigned maxLayers = 8;
+
 public:
-    size_t size() const
+    size_type size() const
     {
-        return size_;
+        return numAttrsInChain;
     }
 
     bool empty() const
     {
-        return !size_;
+        return size() == 0;
     }
 
     class iterator
@@ -94,77 +136,276 @@ public:
         friend class Bindings;
 
     private:
-        pointer ptr = nullptr;
+        struct BindingsCursor
-
-        explicit iterator(pointer ptr)
-            : ptr(ptr)
         {
+            /**
+             * Attr that the cursor currently points to.
+             */
+            pointer current;
+
+            /**
+             * One past the end pointer to the contiguous buffer of Attrs.
+             */
+            pointer end;
+
+            /**
+             * Priority of the value. Lesser values have more priority (i.e. they override
+             * attributes that appear later in the linked list of Bindings).
+             */
+            uint32_t priority;
+
+            pointer operator->() const noexcept
+            {
+                return current;
+            }
+
+            reference get() const noexcept
+            {
+                return *current;
+            }
+
+            bool empty() const noexcept
+            {
+                return current == end;
+            }
+
+            void increment() noexcept
+            {
+                ++current;
+            }
+
+            void consume(Symbol name) noexcept
+            {
+                while (!empty() && current->name <= name)
+                    ++current;
+            }
+
+            GENERATE_CMP(BindingsCursor, me->current->name, me->priority)
+        };
+
+        using QueueStorageType = boost::container::static_vector<BindingsCursor, maxLayers>;
+
+        /**
+         * Comparator implementing the override priority / name ordering
+         * for BindingsCursor.
+         */
+        static constexpr auto comp = std::greater<BindingsCursor>();
+
+        /**
+         * A priority queue used to implement an on-the-fly k-way merge.
+         */
+        QueueStorageType cursorHeap;
+
+        /**
+         * The attribute the iterator currently points to.
+         */
+        pointer current = nullptr;
+
+        /**
+         * Whether iterating over a single attribute and not a merge chain.
+         */
+        bool doMerge = true;
+
+        void push(BindingsCursor cursor) noexcept
+        {
+            cursorHeap.push_back(cursor);
+            std::ranges::make_heap(cursorHeap, comp);
+        }
+
+        [[nodiscard]] BindingsCursor pop() noexcept
+        {
+            std::ranges::pop_heap(cursorHeap, comp);
+            auto cursor = cursorHeap.back();
+            cursorHeap.pop_back();
+            return cursor;
+        }
+
+        iterator & finished() noexcept
+        {
+            current = nullptr;
+            return *this;
+        }
+
+        void next(BindingsCursor cursor) noexcept
+        {
+            current = &cursor.get();
+            cursor.increment();
+
+            if (!cursor.empty())
+                push(cursor);
+        }
+
+        std::optional<BindingsCursor> consumeAllUntilCurrentName() noexcept
+        {
+            auto cursor = pop();
+            Symbol lastHandledName = current->name;
+
+            while (cursor->name <= lastHandledName) {
+                cursor.consume(lastHandledName);
+                if (!cursor.empty())
+                    push(cursor);
+
+                if (cursorHeap.empty())
+                    return std::nullopt;
+
+                cursor = pop();
+            }
+
+            return cursor;
+        }
+
+        explicit iterator(const Bindings & attrs) noexcept
+            : doMerge(attrs.baseLayer)
+        {
+            auto pushBindings = [this, priority = unsigned{0}](const Bindings & layer) mutable {
+                auto first = layer.attrs;
+                push(
+                    BindingsCursor{
+                        .current = first,
+                        .end = first + layer.numAttrs,
+                        .priority = priority++,
+                    });
+            };
+
+            if (!doMerge) {
+                if (attrs.empty())
+                    return;
+
+                current = attrs.attrs;
+                pushBindings(attrs);
+
+                return;
+            }
+
+            const Bindings * layer = &attrs;
+            while (layer) {
+                if (layer->numAttrs != 0)
+                    pushBindings(*layer);
+                layer = layer->baseLayer;
+            }
+
+            if (cursorHeap.empty())
+                return;
+
+            next(pop());
         }
 
     public:
         iterator() = default;
 
-        reference operator*() const
+        reference operator*() const noexcept
         {
-            return *ptr;
+            return *current;
         }
 
-        const value_type * operator->() const
+        pointer operator->() const noexcept
         {
-            return ptr;
+            return current;
         }
 
-        iterator & operator++()
+        iterator & operator++() noexcept
         {
-            ++ptr;
+            if (!doMerge) {
+                ++current;
+                if (current == cursorHeap.front().end)
+                    return finished();
                 return *this;
             }
 
-        iterator operator++(int)
+            if (cursorHeap.empty())
-        {
+                return finished();
-            pointer tmp = ptr;
+
-            ++*this;
+            auto cursor = consumeAllUntilCurrentName();
-            return iterator(tmp);
+            if (!cursor)
+                return finished();
+
+            next(*cursor);
+            return *this;
         }
 
-        bool operator==(const iterator & rhs) const = default;
+        iterator operator++(int) noexcept
+        {
+            iterator tmp = *this;
+            ++*this;
+            return tmp;
+        }
+
+        bool operator==(const iterator & rhs) const noexcept
+        {
+            return current == rhs.current;
+        }
     };
 
     using const_iterator = iterator;
 
     void push_back(const Attr & attr)
     {
-        attrs[size_++] = attr;
+        attrs[numAttrs++] = attr;
+        numAttrsInChain = numAttrs;
     }
 
-    const Attr * get(Symbol name) const
+    /**
+     * Get attribute by name or nullptr if no such attribute exists.
+     */
+    const Attr * get(Symbol name) const noexcept
     {
-        Attr key(name, 0);
+        auto getInChunk = [key = Attr{name, nullptr}](const Bindings & chunk) -> const Attr * {
-        auto first = attrs;
+            auto first = chunk.attrs;
-        auto last = attrs + size_;
+            auto last = first + chunk.numAttrs;
             const Attr * i = std::lower_bound(first, last, key);
-        if (i != last && i->name == name)
+            if (i != last && i->name == key.name)
                 return i;
             return nullptr;
+        };
+
+        const Bindings * currentChunk = this;
+        while (currentChunk) {
+            const Attr * maybeAttr = getInChunk(*currentChunk);
+            if (maybeAttr)
+                return maybeAttr;
+            currentChunk = currentChunk->baseLayer;
+        }
+
+        return nullptr;
+    }
+
+    /**
+     * Check if the layer chain is full.
+     */
+    bool isLayerListFull() const noexcept
+    {
+        return numLayers == Bindings::maxLayers;
+    }
+
+    /**
+     * Test if the length of the linked list of layers is greater than 1.
+     */
+    bool isLayered() const noexcept
+    {
+        return numLayers > 1;
     }
 
     const_iterator begin() const
     {
-        return const_iterator(attrs);
+        return const_iterator(*this);
     }
 
     const_iterator end() const
     {
-        return const_iterator(attrs + size_);
+        return const_iterator();
     }
 
-    Attr & operator[](size_t pos)
+    Attr & operator[](size_type pos)
     {
+        if (isLayered()) [[unlikely]]
+            unreachable();
         return attrs[pos];
     }
 
-    const Attr & operator[](size_t pos) const
+    const Attr & operator[](size_type pos) const
     {
+        if (isLayered()) [[unlikely]]
+            unreachable();
         return attrs[pos];
     }
 
@@ -176,17 +417,16 @@ public:
     std::vector<const Attr *> lexicographicOrder(const SymbolTable & symbols) const
     {
         std::vector<const Attr *> res;
-        res.reserve(size_);
+        res.reserve(size());
-        for (size_t n = 0; n < size_; n++)
+        std::ranges::transform(*this, std::back_inserter(res), [](const Attr & a) { return &a; });
-            res.emplace_back(&attrs[n]);
+        std::ranges::sort(res, [&](const Attr * a, const Attr * b) {
-        std::sort(res.begin(), res.end(), [&](const Attr * a, const Attr * b) {
             std::string_view sa = symbols[a->name], sb = symbols[b->name];
             return sa < sb;
         });
         return res;
     }
 
-    friend class EvalState;
+    friend class EvalMemory;
 };
 
 static_assert(std::forward_iterator<Bindings::iterator>);
@@ -202,23 +442,38 @@ class BindingsBuilder final
 public:
     // needed by std::back_inserter
     using value_type = Attr;
-    using size_type = Bindings::size_t;
+    using size_type = Bindings::size_type;
 
 private:
     Bindings * bindings;
-    Bindings::size_t capacity_;
+    Bindings::size_type capacity_;
 
-    friend class EvalState;
+    friend class EvalMemory;
 
-    BindingsBuilder(EvalState & state, Bindings * bindings, size_type capacity)
+    BindingsBuilder(EvalMemory & mem, SymbolTable & symbols, Bindings * bindings, size_type capacity)
         : bindings(bindings)
         , capacity_(capacity)
-        , state(state)
+        , mem(mem)
+        , symbols(symbols)
     {
     }
 
+    bool hasBaseLayer() const noexcept
+    {
+        return bindings->baseLayer;
+    }
+
+    void finishSizeIfNecessary()
+    {
+        if (hasBaseLayer())
+            /* NOTE: Do not use std::ranges::distance, since Bindings is a sized
+               range, but we are calculating this size here. */
+            bindings->numAttrsInChain = std::distance(bindings->begin(), bindings->end());
+    }
+
 public:
-    std::reference_wrapper<EvalState> state;
+    std::reference_wrapper<EvalMemory> mem;
+    std::reference_wrapper<SymbolTable> symbols;
 
     void insert(Symbol name, Value * value, PosIdx pos = noPos)
     {
@@ -232,10 +487,26 @@ public:
 
     void push_back(const Attr & attr)
     {
-        assert(bindings->size() < capacity_);
+        assert(bindings->numAttrs < capacity_);
         bindings->push_back(attr);
     }
 
+    /**
+     * "Layer" the newly constructured Bindings on top of another attribute set.
+     *
+     * This effectively performs an attribute set merge, while giving preference
+     * to attributes from the newly constructed Bindings in case of duplicate attribute
+     * names.
+     *
+     * This operation amortizes the need to copy over all attributes and allows
+     * for efficient implementation of attribute set merges (ExprOpUpdate::eval).
+     */
+    void layerOnTopOf(const Bindings & base) noexcept
+    {
+        bindings->baseLayer = &base;
+        bindings->numLayers = base.numLayers + 1;
+    }
+
     Value & alloc(Symbol name, PosIdx pos = noPos);
 
     Value & alloc(std::string_view name, PosIdx pos = noPos);
@@ -243,11 +514,13 @@ public:
     Bindings * finish()
     {
         bindings->sort();
+        finishSizeIfNecessary();
         return bindings;
     }
 
     Bindings * alreadySorted()
     {
+        finishSizeIfNecessary();
         return bindings;
     }
 

src/libexpr/include/nix/expr/counter.hh

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <atomic>
+#include <cstdint>
+
+namespace nix {
+
+/**
+ * An atomic counter aligned on a cache line to prevent false sharing.
+ * The counter is only enabled when the `NIX_SHOW_STATS` environment
+ * variable is set. This is to prevent contention on these counters
+ * when multi-threaded evaluation is enabled.
+ */
+struct alignas(64) Counter
+{
+    using value_type = uint64_t;
+
+    std::atomic<value_type> inner{0};
+
+    static bool enabled;
+
+    Counter() {}
+
+    operator value_type() const noexcept
+    {
+        return inner;
+    }
+
+    void operator=(value_type n) noexcept
+    {
+        inner = n;
+    }
+
+    value_type load() const noexcept
+    {
+        return inner;
+    }
+
+    value_type operator++() noexcept
+    {
+        return enabled ? ++inner : 0;
+    }
+
+    value_type operator++(int) noexcept
+    {
+        return enabled ? inner++ : 0;
+    }
+
+    value_type operator--() noexcept
+    {
+        return enabled ? --inner : 0;
+    }
+
+    value_type operator--(int) noexcept
+    {
+        return enabled ? inner-- : 0;
+    }
+
+    value_type operator+=(value_type n) noexcept
+    {
+        return enabled ? inner += n : 0;
+    }
+
+    value_type operator-=(value_type n) noexcept
+    {
+        return enabled ? inner -= n : 0;
+    }
+};
+
+} // namespace nix

src/libexpr/include/nix/expr/eval-inline.hh

@@ -26,7 +26,7 @@ inline void * allocBytes(size_t n)
 }
 
 [[gnu::always_inline]]
-Value * EvalState::allocValue()
+Value * EvalMemory::allocValue()
 {
 #if NIX_USE_BOEHMGC
     /* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
@@ -48,15 +48,15 @@ Value * EvalState::allocValue()
     void * p = allocBytes(sizeof(Value));
 #endif
 
-    nrValues++;
+    stats.nrValues++;
     return (Value *) p;
 }
 
 [[gnu::always_inline]]
-Env & EvalState::allocEnv(size_t size)
+Env & EvalMemory::allocEnv(size_t size)
 {
-    nrEnvs++;
+    stats.nrEnvs++;
-    nrValuesInEnvs += size;
+    stats.nrValuesInEnvs += size;
 
     Env * env;
 

src/libexpr/include/nix/expr/eval-settings.hh

@@ -342,6 +342,25 @@ struct EvalSettings : Config
           This is useful for improving code readability and making path literals
           more explicit.
     )"};
+
+    Setting<unsigned> bindingsUpdateLayerRhsSizeThreshold{
+        this,
+        sizeof(void *) == 4 ? 8192 : 16,
+        "eval-attrset-update-layer-rhs-threshold",
+        R"(
+          Tunes the maximum size of an attribute set that, when used
+          as a right operand in an [attribute set update expression](@docroot@/language/operators.md#update),
+          uses a more space-efficient linked-list representation of attribute sets.
+
+          Setting this to larger values generally leads to less memory allocations,
+          but may lead to worse evaluation performance.
+
+          A value of `0` disables this optimization completely.
+
+          This is an advanced performance tuning option and typically should not be changed.
+          The default value is chosen to balance performance and memory usage. On 32 bit systems
+          where memory is scarce, the default is a large value to reduce the amount of allocations.
+    )"};
 };
 
 /**

src/libexpr/include/nix/expr/eval.hh

@@ -16,6 +16,7 @@
 #include "nix/expr/search-path.hh"
 #include "nix/expr/repl-exit-status.hh"
 #include "nix/util/ref.hh"
+#include "nix/expr/counter.hh"
 
 // For `NIX_USE_BOEHMGC`, and if that's set, `GC_THREADS`
 #include "nix/expr/config.hh"
@@ -48,6 +49,7 @@ class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
 struct MemorySourceAccessor;
+struct MountedSourceAccessor;
 
 namespace eval_cache {
 class EvalCache;
@@ -300,6 +302,68 @@ struct StaticEvalSymbols
     }
 };
 
+class EvalMemory
+{
+#if NIX_USE_BOEHMGC
+    /**
+     * Allocation cache for GC'd Value objects.
+     */
+    std::shared_ptr<void *> valueAllocCache;
+
+    /**
+     * Allocation cache for size-1 Env objects.
+     */
+    std::shared_ptr<void *> env1AllocCache;
+#endif
+
+public:
+    struct Statistics
+    {
+        Counter nrEnvs;
+        Counter nrValuesInEnvs;
+        Counter nrValues;
+        Counter nrAttrsets;
+        Counter nrAttrsInAttrsets;
+        Counter nrListElems;
+    };
+
+    EvalMemory();
+
+    EvalMemory(const EvalMemory &) = delete;
+    EvalMemory(EvalMemory &&) = delete;
+    EvalMemory & operator=(const EvalMemory &) = delete;
+    EvalMemory & operator=(EvalMemory &&) = delete;
+
+    inline Value * allocValue();
+    inline Env & allocEnv(size_t size);
+
+    Bindings * allocBindings(size_t capacity);
+
+    BindingsBuilder buildBindings(SymbolTable & symbols, size_t capacity)
+    {
+        return BindingsBuilder(*this, symbols, allocBindings(capacity), capacity);
+    }
+
+    ListBuilder buildList(size_t size)
+    {
+        stats.nrListElems += size;
+        return ListBuilder(size);
+    }
+
+    const Statistics & getStats() const &
+    {
+        return stats;
+    }
+
+    /**
+     * Storage for the AST nodes
+     */
+    Exprs exprs;
+
+private:
+    Statistics stats;
+};
+
 class EvalState : public std::enable_shared_from_this<EvalState>
 {
 public:
@@ -310,6 +374,8 @@ public:
     SymbolTable symbols;
     PosTable positions;
 
+    EvalMemory mem;
+
     /**
      * If set, force copying files to the Nix store even if they
      * already exist there.
@@ -319,7 +385,7 @@ public:
     /**
      * The accessor corresponding to `store`.
      */
-    const ref<SourceAccessor> storeFS;
+    const ref<MountedSourceAccessor> storeFS;
 
     /**
      * The accessor for the root filesystem.
@@ -439,18 +505,6 @@ private:
      */
     std::shared_ptr<RegexCache> regexCache;
 
-#if NIX_USE_BOEHMGC
-    /**
-     * Allocation cache for GC'd Value objects.
-     */
-    std::shared_ptr<void *> valueAllocCache;
-
-    /**
-     * Allocation cache for size-1 Env objects.
-     */
-    std::shared_ptr<void *> env1AllocCache;
-#endif
-
 public:
 
     EvalState(
@@ -461,6 +515,15 @@ public:
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
+    /**
+     * A wrapper around EvalMemory::allocValue() to avoid code churn when it
+     * was introduced.
+     */
+    inline Value * allocValue()
+    {
+        return mem.allocValue();
+    }
+
     LookupPath getLookupPath()
     {
         return lookupPath;
@@ -488,8 +551,11 @@ public:
 
     /**
      * Allow access to a path.
+     *
+     * Only for restrict eval: pure eval just whitelist store paths,
+     * never arbitrary paths.
      */
-    void allowPath(const Path & path);
+    void allowPathLegacy(const Path & path);
 
     /**
      * Allow access to a store path. Note that this gets remapped to
@@ -829,22 +895,14 @@ public:
      */
     void autoCallFunction(const Bindings & args, Value & fun, Value & res);
 
-    /**
-     * Allocation primitives.
-     */
-    inline Value * allocValue();
-    inline Env & allocEnv(size_t size);
-
-    Bindings * allocBindings(size_t capacity);
-
     BindingsBuilder buildBindings(size_t capacity)
     {
-        return BindingsBuilder(*this, allocBindings(capacity), capacity);
+        return mem.buildBindings(symbols, capacity);
     }
 
     ListBuilder buildList(size_t size)
     {
-        return ListBuilder(*this, size);
+        return mem.buildList(size);
     }
 
     /**
@@ -961,19 +1019,13 @@ private:
      */
     std::string mkSingleDerivedPathStringRaw(const SingleDerivedPath & p);
 
-    unsigned long nrEnvs = 0;
+    Counter nrLookups;
-    unsigned long nrValuesInEnvs = 0;
+    Counter nrAvoided;
-    unsigned long nrValues = 0;
+    Counter nrOpUpdates;
-    unsigned long nrListElems = 0;
+    Counter nrOpUpdateValuesCopied;
-    unsigned long nrLookups = 0;
+    Counter nrListConcats;
-    unsigned long nrAttrsets = 0;
+    Counter nrPrimOpCalls;
-    unsigned long nrAttrsInAttrsets = 0;
+    Counter nrFunctionCalls;
-    unsigned long nrAvoided = 0;
-    unsigned long nrOpUpdates = 0;
-    unsigned long nrOpUpdateValuesCopied = 0;
-    unsigned long nrListConcats = 0;
-    unsigned long nrPrimOpCalls = 0;
-    unsigned long nrFunctionCalls = 0;
 
     bool countCalls;
 

src/libexpr/include/nix/expr/gc-small-vector.hh

@@ -26,4 +26,20 @@ using SmallValueVector = SmallVector<Value *, nItems>;
 template<size_t nItems>
 using SmallTemporaryValueVector = SmallVector<Value, nItems>;
 
+/**
+ * For functions where we do not expect deep recursion, we can use a sizable
+ * part of the stack a free allocation space.
+ *
+ * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
+ */
+constexpr size_t nonRecursiveStackReservation = 128;
+
+/**
+ * Functions that maybe applied to self-similar inputs, such as concatMap on a
+ * tree, should reserve a smaller part of the stack for allocation.
+ *
+ * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
+ */
+constexpr size_t conservativeStackReservation = 16;
+
 } // namespace nix

src/libexpr/include/nix/expr/meson.build

@@ -10,6 +10,7 @@ config_pub_h = configure_file(
 headers = [ config_pub_h ] + files(
   'attr-path.hh',
   'attr-set.hh',
+  'counter.hh',
   'eval-cache.hh',
   'eval-error.hh',
   'eval-gc.hh',

src/libexpr/include/nix/expr/nixexpr.hh

@@ -3,11 +3,14 @@
 
 #include <map>
 #include <vector>
+#include <memory_resource>
 
+#include "nix/expr/gc-small-vector.hh"
 #include "nix/expr/value.hh"
 #include "nix/expr/symbol-table.hh"
 #include "nix/expr/eval-error.hh"
 #include "nix/util/pos-idx.hh"
+#include "nix/expr/counter.hh"
 
 namespace nix {
 
@@ -80,6 +83,15 @@ typedef std::vector<AttrName> AttrPath;
 
 std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath);
 
+using UpdateQueue = SmallTemporaryValueVector<conservativeStackReservation>;
+
+class Exprs
+{
+    std::pmr::monotonic_buffer_resource buffer;
+public:
+    std::pmr::polymorphic_allocator<char> alloc{&buffer};
+};
+
 /* Abstract syntax of Nix expressions. */
 
 struct Expr
@@ -89,7 +101,7 @@ struct Expr
         Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
     };
 
-    static unsigned long nrExprs;
+    static Counter nrExprs;
 
     Expr()
     {
@@ -99,8 +111,25 @@ struct Expr
     virtual ~Expr() {};
     virtual void show(const SymbolTable & symbols, std::ostream & str) const;
     virtual void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+
+    /** Normal evaluation, implemented directly by all subclasses. */
     virtual void eval(EvalState & state, Env & env, Value & v);
+
+    /**
+     * Create a thunk for the delayed computation of the given expression
+     * in the given environment. But if the expression is a variable,
+     * then look it up right away. This significantly reduces the number
+     * of thunks allocated.
+     */
     virtual Value * maybeThunk(EvalState & state, Env & env);
+
+    /**
+     * Only called when performing an attrset update: `//` or similar.
+     * Instead of writing to a Value &, this function writes to an UpdateQueue.
+     * This allows the expression to perform multiple updates in a delayed manner, gathering up all the updates before
+     * applying them.
+     */
+    virtual void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx);
     virtual void setName(Symbol name);
     virtual void setDocComment(DocComment docComment) {};
 
@@ -152,13 +181,28 @@ struct ExprFloat : Expr
 
 struct ExprString : Expr
 {
-    std::string s;
     Value v;
 
-    ExprString(std::string && s)
+    /**
-        : s(std::move(s))
+     * This is only for strings already allocated in our polymorphic allocator,
+     * or that live at least that long (e.g. c++ string literals)
+     */
+    ExprString(const char * s)
     {
-        v.mkStringNoCopy(this->s.data());
+        v.mkStringNoCopy(s);
+    };
+
+    ExprString(std::pmr::polymorphic_allocator<char> & alloc, std::string_view sv)
+    {
+        auto len = sv.length();
+        if (len == 0) {
+            v.mkStringNoCopy("");
+            return;
+        }
+        char * s = alloc.allocate(len + 1);
+        sv.copy(s, len);
+        s[len] = '\0';
+        v.mkStringNoCopy(s);
     };
 
     Value * maybeThunk(EvalState & state, Env & env) override;
@@ -565,18 +609,16 @@ struct ExprOpNot : Expr
     COMMON_METHODS
 };
 
-#define MakeBinOp(name, s)                                                                   \
+#define MakeBinOpMembers(name, s)                                                        \
-    struct name : Expr                                                                       \
-    {                                                                                        \
     PosIdx pos;                                                                          \
     Expr *e1, *e2;                                                                       \
     name(Expr * e1, Expr * e2)                                                           \
         : e1(e1)                                                                         \
-            , e2(e2) {};                                                                     \
+        , e2(e2){};                                                                      \
     name(const PosIdx & pos, Expr * e1, Expr * e2)                                       \
         : pos(pos)                                                                       \
         , e1(e1)                                                                         \
-            , e2(e2) {};                                                                     \
+        , e2(e2){};                                                                      \
     void show(const SymbolTable & symbols, std::ostream & str) const override            \
     {                                                                                    \
         str << "(";                                                                      \
@@ -594,7 +636,12 @@ struct ExprOpNot : Expr
     PosIdx getPos() const override                                                       \
     {                                                                                    \
         return pos;                                                                      \
-        }                                                                                    \
+    }
+
+#define MakeBinOp(name, s)        \
+    struct name : Expr            \
+    {                             \
+        MakeBinOpMembers(name, s) \
     }
 
 MakeBinOp(ExprOpEq, "==");
@@ -602,9 +649,20 @@ MakeBinOp(ExprOpNEq, "!=");
 MakeBinOp(ExprOpAnd, "&&");
 MakeBinOp(ExprOpOr, "||");
 MakeBinOp(ExprOpImpl, "->");
-MakeBinOp(ExprOpUpdate, "//");
 MakeBinOp(ExprOpConcatLists, "++");
 
+struct ExprOpUpdate : Expr
+{
+private:
+    /** Special case for merging of two attrsets. */
+    void eval(EvalState & state, Value & v, Value & v1, Value & v2);
+    void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q);
+
+public:
+    MakeBinOpMembers(ExprOpUpdate, "//");
+    virtual void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx) override;
+};
+
 struct ExprConcatStrings : Expr
 {
     PosIdx pos;

src/libexpr/include/nix/expr/parser-state.hh

@@ -24,7 +24,6 @@ struct StringToken
     }
 };
 
-// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y.
 struct ParserLocation
 {
     int beginOffset;
@@ -44,9 +43,6 @@ struct ParserLocation
         beginOffset = stashedBeginOffset;
         endOffset = stashedEndOffset;
     }
-
-    /** Latest doc comment position, or 0. */
-    int doc_comment_first_column, doc_comment_last_column;
 };
 
 struct LexerState
@@ -82,6 +78,7 @@ struct LexerState
 struct ParserState
 {
     const LexerState & lexerState;
+    std::pmr::polymorphic_allocator<char> & alloc;
     SymbolTable & symbols;
     PosTable & positions;
     Expr * result;
@@ -327,7 +324,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
 
         // Ignore empty strings for a minor optimisation and AST simplification
         if (s2 != "") {
-            es2->emplace_back(i->first, new ExprString(std::move(s2)));
+            es2->emplace_back(i->first, new ExprString(alloc, s2));
         }
     };
     for (; i != es.end(); ++i, --n) {

src/libexpr/include/nix/expr/primops.hh

@@ -8,22 +8,6 @@
 
 namespace nix {
 
-/**
- * For functions where we do not expect deep recursion, we can use a sizable
- * part of the stack a free allocation space.
- *
- * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
- */
-constexpr size_t nonRecursiveStackReservation = 128;
-
-/**
- * Functions that maybe applied to self-similar inputs, such as concatMap on a
- * tree, should reserve a smaller part of the stack for allocation.
- *
- * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
- */
-constexpr size_t conservativeStackReservation = 16;
-
 struct RegisterPrimOp
 {
     typedef std::vector<PrimOp> PrimOps;

src/libexpr/include/nix/expr/value.hh

@@ -155,7 +155,7 @@ class ListBuilder
     Value * inlineElems[2] = {nullptr, nullptr};
 public:
     Value ** elems;
-    ListBuilder(EvalState & state, size_t size);
+    ListBuilder(size_t size);
 
     // NOTE: Can be noexcept because we are just copying integral values and
     // raw pointers.

src/libexpr/lexer-helpers.cc

@@ -1,11 +1,11 @@
 #include "lexer-helpers.hh"
 
-void nix::lexer::internal::initLoc(YYLTYPE * loc)
+void nix::lexer::internal::initLoc(Parser::location_type * loc)
 {
     loc->beginOffset = loc->endOffset = 0;
 }
 
-void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
+void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, Parser::location_type * loc, const char * s, size_t len)
 {
     loc->stash();
 

src/libexpr/lexer-helpers.hh

@@ -2,16 +2,12 @@
 
 #include <cstddef>
 
-// including the generated headers twice leads to errors
+#include "parser-scanner-decls.hh"
-#ifndef BISON_HEADER
-#  include "lexer-tab.hh"
-#  include "parser-tab.hh"
-#endif
 
 namespace nix::lexer::internal {
 
-void initLoc(YYLTYPE * loc);
+void initLoc(Parser::location_type * loc);
 
-void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
+void adjustLoc(yyscan_t yyscanner, Parser::location_type * loc, const char * s, size_t len);
 
 } // namespace nix::lexer::internal

src/libexpr/lexer.l

@@ -82,6 +82,10 @@ static void requireExperimentalFeature(const ExperimentalFeature & feature, cons
 
 }
 
+using enum nix::Parser::token::token_kind_type;
+using YYSTYPE = nix::Parser::value_type;
+using YYLTYPE = nix::Parser::location_type;
+
 // yacc generates code that uses unannotated fallthrough.
 #pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
 

src/libexpr/meson.build

@@ -53,7 +53,12 @@ deps_other += boost
 nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
-bdw_gc = dependency('bdw-gc', required : get_option('gc'))
+bdw_gc_required = get_option('gc').disable_if(
+  'address' in get_option('b_sanitize'),
+  error_message : 'Building with Boehm GC and ASAN is not supported',
+)
+
+bdw_gc = dependency('bdw-gc', required : bdw_gc_required)
 if bdw_gc.found()
   deps_public += bdw_gc
   foreach funcspec : [
@@ -64,6 +69,10 @@ if bdw_gc.found()
     define_value = cxx.has_function(funcspec).to_int()
     configdata_priv.set(define_name, define_value)
   endforeach
+  if host_machine.system() == 'cygwin'
+    # undefined reference to `__wrap__Znwm'
+    configdata_pub.set('GC_NO_INLINE_STD_NEW', 1)
+  endif
 endif
 # Used in public header. Affects ABI!
 configdata_pub.set('NIX_USE_BOEHMGC', bdw_gc.found().to_int())
@@ -88,6 +97,7 @@ config_priv_h = configure_file(
 )
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 parser_tab = custom_target(
   input : 'parser.y',

src/libexpr/nixexpr.cc

@@ -11,7 +11,7 @@
 
 namespace nix {
 
-unsigned long Expr::nrExprs = 0;
+Counter Expr::nrExprs;
 
 ExprBlackHole eBlackHole;
 
@@ -40,7 +40,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    printLiteralString(str, s);
+    printLiteralString(str, v.string_view());
 }
 
 void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const

src/libexpr/parser-scanner-decls.hh

@@ -0,0 +1,17 @@
+#pragma once
+
+#ifndef BISON_HEADER
+#  include "parser-tab.hh"
+using YYSTYPE = nix::parser::BisonParser::value_type;
+using YYLTYPE = nix::parser::BisonParser::location_type;
+#  include "lexer-tab.hh" // IWYU pragma: export
+#endif
+
+namespace nix {
+
+class Parser : public parser::BisonParser
+{
+    using BisonParser::BisonParser;
+};
+
+} // namespace nix

src/libexpr/parser.y

@@ -1,5 +1,7 @@
+%skeleton "lalr1.cc"
 %define api.location.type { ::nix::ParserLocation }
-%define api.pure
+%define api.namespace { ::nix::parser }
+%define api.parser.class { BisonParser }
 %locations
 %define parse.error verbose
 %defines
@@ -26,19 +28,12 @@
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/parser-state.hh"
 
-// Bison seems to have difficulty growing the parser stack when using C++ with
+#define YY_DECL                                    \
-// a custom location type. This undocumented macro tells Bison that our
+    int yylex(                                     \
-// location type is "trivially copyable" in C++-ese, so it is safe to use the
+        nix::Parser::value_type * yylval_param,    \
-// same memcpy macro it uses to grow the stack that it uses with its own
+        nix::Parser::location_type * yylloc_param, \
-// default location type. Without this, we get "error: memory exhausted" when
+        yyscan_t yyscanner,                        \
-// parsing some large Nix files. Our other options are to increase the initial
+        nix::ParserState * state)
-// stack size (200 by default) to be as large as we ever want to support (so
-// that growing the stack is unnecessary), or redefine the stack-relocation
-// macro ourselves (which is also undocumented).
-#define YYLTYPE_IS_TRIVIAL 1
-
-#define YY_DECL int yylex \
-    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
 
 // For efficiency, we only track offsets; not line,column coordinates
 # define YYLLOC_DEFAULT(Current, Rhs, N)                                \
@@ -64,6 +59,7 @@ Expr * parseExprFromBuf(
     size_t length,
     Pos::Origin origin,
     const SourcePath & basePath,
+    std::pmr::polymorphic_allocator<char> & alloc,
     SymbolTable & symbols,
     const EvalSettings & settings,
     PosTable & positions,
@@ -78,24 +74,30 @@ Expr * parseExprFromBuf(
 
 %{
 
-#include "parser-tab.hh"
+/* The parser is very performance sensitive and loses out on a lot
-#include "lexer-tab.hh"
+   of performance even with basic stdlib assertions. Since those don't
+   affect ABI we can disable those just for this file. */
+#if defined(_GLIBCXX_ASSERTIONS) && !defined(_GLIBCXX_DEBUG)
+#undef _GLIBCXX_ASSERTIONS
+#endif
+
+#include "parser-scanner-decls.hh"
 
 YY_DECL;
 
 using namespace nix;
 
-#define CUR_POS state->at(yyloc)
+#define CUR_POS state->at(yylhs.location)
 
-
+void parser::BisonParser::error(const location_type &loc_, const std::string &error)
-void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
 {
+    auto loc = loc_;
     if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
-        loc->beginOffset = loc->endOffset;
+        loc.beginOffset = loc.endOffset;
     }
     throw ParseError({
         .msg = HintFmt(error),
-        .pos = state->positions[state->at(*loc)]
+        .pos = state->positions[state->at(loc)]
     });
 }
 
@@ -134,6 +136,7 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
   std::vector<nix::AttrName> * attrNames;
   std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
   std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
+  std::variant<nix::Expr *, std::string_view> * to_be_string;
   std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
 }
 
@@ -148,7 +151,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
 %type <inheritAttrs> attrs
 %type <string_parts> string_parts_interpolated
 %type <ind_string_parts> ind_string_parts
-%type <e> path_start string_parts string_attr
+%type <e> path_start
+%type <to_be_string> string_parts string_attr
 %type <id> attr
 %token <id> ID
 %token <str> STR IND_STR
@@ -182,7 +186,7 @@ start: expr {
   state->result = $1;
 
   // This parser does not use yynerrs; suppress the warning.
-  (void) yynerrs;
+  (void) yynerrs_;
 };
 
 expr: expr_function;
@@ -303,7 +307,13 @@ expr_simple
   }
   | INT_LIT { $$ = new ExprInt($1); }
   | FLOAT_LIT { $$ = new ExprFloat($1); }
-  | '"' string_parts '"' { $$ = $2; }
+  | '"' string_parts '"' {
+      std::visit(overloaded{
+          [&](std::string_view str) { $$ = new ExprString(state->alloc, str); },
+          [&](Expr * expr) { $$ = expr; }},
+      *$2);
+      delete $2;
+  }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
       $$ = state->stripIndentation(CUR_POS, std::move(*$2));
       delete $2;
@@ -314,11 +324,11 @@ expr_simple
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
-      std::string path($1.p + 1, $1.l - 2);
+      std::string_view path($1.p + 1, $1.l - 2);
       $$ = new ExprCall(CUR_POS,
           new ExprVar(state->s.findFile),
           {new ExprVar(state->s.nixPath),
-           new ExprString(std::move(path))});
+           new ExprString(state->alloc, path)});
   }
   | URI {
       static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
@@ -327,7 +337,7 @@ expr_simple
               .msg = HintFmt("URL literals are disabled"),
               .pos = state->positions[CUR_POS]
           });
-      $$ = new ExprString(std::string($1));
+      $$ = new ExprString(state->alloc, $1);
   }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
@@ -344,19 +354,19 @@ expr_simple
   ;
 
 string_parts
-  : STR { $$ = new ExprString(std::string($1)); }
+  : STR { $$ = new std::variant<Expr *, std::string_view>($1); }
-  | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); }
+  | string_parts_interpolated { $$ = new std::variant<Expr *, std::string_view>(new ExprConcatStrings(CUR_POS, true, $1)); }
-  | { $$ = new ExprString(""); }
+  | { $$ = new std::variant<Expr *, std::string_view>(std::string_view()); }
   ;
 
 string_parts_interpolated
   : string_parts_interpolated STR
-  { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); }
+  { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(state->alloc, $2)); }
   | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
   | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
   | STR DOLLAR_CURLY expr '}' {
       $$ = new std::vector<std::pair<PosIdx, Expr *>>;
-      $$->emplace_back(state->at(@1), new ExprString(std::string($1)));
+      $$->emplace_back(state->at(@1), new ExprString(state->alloc, $1));
       $$->emplace_back(state->at(@2), $3);
     }
   ;
@@ -454,16 +464,17 @@ attrs
   : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
   | attrs string_attr
     { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($2);
+      std::visit(overloaded {
-      if (str) {
+          [&](std::string_view str) { $$->emplace_back(AttrName(state->symbols.create(str)), state->at(@2)); },
-          $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2));
+          [&](Expr * expr) {
-          delete str;
-      } else
               throw ParseError({
                   .msg = HintFmt("dynamic attributes not allowed in inherit"),
                   .pos = state->positions[state->at(@2)]
               });
           }
+      }, *$2);
+      delete $2;
+    }
   | { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
   ;
 
@@ -471,22 +482,20 @@ attrpath
   : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
   | attrpath '.' string_attr
     { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($3);
+      std::visit(overloaded {
-      if (str) {
+          [&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
-          $$->push_back(AttrName(state->symbols.create(str->s)));
+          [&](Expr * expr) { $$->push_back(AttrName(expr)); }
-          delete str;
+      }, *$3);
-      } else
+      delete $3;
-          $$->push_back(AttrName($3));
     }
   | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
   | string_attr
     { $$ = new std::vector<AttrName>;
-      ExprString *str = dynamic_cast<ExprString *>($1);
+      std::visit(overloaded {
-      if (str) {
+          [&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
-          $$->push_back(AttrName(state->symbols.create(str->s)));
+          [&](Expr * expr) { $$->push_back(AttrName(expr)); }
-          delete str;
+      }, *$1);
-      } else
+      delete $1;
-          $$->push_back(AttrName($1));
     }
   ;
 
@@ -497,7 +506,7 @@ attr
 
 string_attr
   : '"' string_parts '"' { $$ = $2; }
-  | DOLLAR_CURLY expr '}' { $$ = $2; }
+  | DOLLAR_CURLY expr '}' { $$ = new std::variant<Expr *, std::string_view>($2); }
   ;
 
 expr_list
@@ -537,6 +546,7 @@ Expr * parseExprFromBuf(
     size_t length,
     Pos::Origin origin,
     const SourcePath & basePath,
+    std::pmr::polymorphic_allocator<char> & alloc,
     SymbolTable & symbols,
     const EvalSettings & settings,
     PosTable & positions,
@@ -551,6 +561,7 @@ Expr * parseExprFromBuf(
     };
     ParserState state {
         .lexerState = lexerState,
+        .alloc = alloc,
         .symbols = symbols,
         .positions = positions,
         .basePath = basePath,
@@ -563,7 +574,8 @@ Expr * parseExprFromBuf(
     Finally _destroy([&] { yylex_destroy(scanner); });
 
     yy_scan_buffer(text, length, scanner);
-    yyparse(scanner, &state);
+    Parser parser(scanner, &state);
+    parser.parse();
 
     return state.result;
 }

src/libexpr/primops.cc

@@ -262,7 +262,7 @@ static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path,
 {
     state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport");
 
-    Env * env = &state.allocEnv(vScope->attrs()->size());
+    Env * env = &state.mem.allocEnv(vScope->attrs()->size());
     env->up = &state.baseEnv;
 
     auto staticEnv = std::make_shared<StaticEnv>(nullptr, state.staticBaseEnv, vScope->attrs()->size());
@@ -3161,7 +3161,7 @@ static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value ** args,
     // Step 1. Sort the name-value attrsets in place using the memory we allocate for the result
     auto listView = args[0]->listView();
     size_t listSize = listView.size();
-    auto & bindings = *state.allocBindings(listSize);
+    auto & bindings = *state.mem.allocBindings(listSize);
     using ElemPtr = decltype(&bindings[0].value);
 
     for (const auto & [n, v2] : enumerate(listView)) {

src/libfetchers-c/meson.build

@@ -32,6 +32,7 @@ add_project_arguments(
 )
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'nix_api_fetchers.cc',

src/libfetchers-tests/git.cc

@@ -1,5 +1,6 @@
 #include "nix/store/store-open.hh"
 #include "nix/store/globals.hh"
+#include "nix/store/dummy-store.hh"
 #include "nix/fetchers/fetch-settings.hh"
 #include "nix/fetchers/fetchers.hh"
 #include "nix/fetchers/git-utils.hh"
@@ -179,10 +180,11 @@ TEST_F(GitTest, submodulePeriodSupport)
     // 6) Commit the addition in super
     commitAll(super.get(), "Add submodule with branch='.'");
 
-    // TODO: Use dummy:// store with MemorySourceAccessor.
+    auto store = [] {
-    Path storeTmpDir = createTempDir();
+        auto cfg = make_ref<DummyStoreConfig>(StoreReference::Params{});
-    auto storeTmpDirAutoDelete = AutoDelete(storeTmpDir, true);
+        cfg->readOnly = false;
-    ref<Store> store = openStore(storeTmpDir);
+        return cfg->openStore();
+    }();
 
     auto settings = fetchers::Settings{};
     auto input = fetchers::Input::fromAttrs(

src/libfetchers-tests/meson.build

@@ -37,6 +37,7 @@ libgit2 = dependency('libgit2')
 deps_private += libgit2
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'access-tokens.cc',
@@ -63,7 +64,7 @@ this_exe = executable(
 test(
   meson.project_name(),
   this_exe,
-  env : {
+  env : asan_test_options_env + {
     '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data',
   },
   protocol : 'gtest',

src/libfetchers-tests/package.nix

@@ -61,6 +61,7 @@ mkMesonExecutable (finalAttrs: {
             buildInputs = [ writableTmpDirAsHomeHook ];
           }
           ''
+            export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0
             export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
             ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
             touch $out

src/libfetchers-tests/public-key.cc

@@ -1,14 +1,14 @@
 #include <gtest/gtest.h>
 #include "nix/fetchers/fetchers.hh"
 #include "nix/util/json-utils.hh"
-#include <nlohmann/json.hpp>
+#include "nix/util/tests/json-characterization.hh"
-#include "nix/util/tests/characterization.hh"
 
 namespace nix {
 
 using nlohmann::json;
 
-class PublicKeyTest : public CharacterizationTest
+class PublicKeyTest : public JsonCharacterizationTest<fetchers::PublicKey>,
+                      public ::testing::WithParamInterface<std::pair<std::string_view, fetchers::PublicKey>>
 {
     std::filesystem::path unitTestData = getUnitTestData() / "public-key";
 
@@ -19,30 +19,35 @@ public:
     }
 };
 
-#define TEST_JSON(FIXTURE, NAME, VAL)                                                                 \
+TEST_P(PublicKeyTest, from_json)
-    TEST_F(FIXTURE, PublicKey_##NAME##_from_json)                                                     \
+{
-    {                                                                                                 \
+    const auto & [name, expected] = GetParam();
-        readTest(#NAME ".json", [&](const auto & encoded_) {                                          \
+    readJsonTest(name, expected);
-            fetchers::PublicKey expected{VAL};                                                        \
+}
-            fetchers::PublicKey got = nlohmann::json::parse(encoded_);                                \
-            ASSERT_EQ(got, expected);                                                                 \
-        });                                                                                           \
-    }                                                                                                 \
-                                                                                                      \
-    TEST_F(FIXTURE, PublicKey_##NAME##_to_json)                                                       \
-    {                                                                                                 \
-        writeTest(                                                                                    \
-            #NAME ".json",                                                                            \
-            [&]() -> json { return nlohmann::json(fetchers::PublicKey{VAL}); },                       \
-            [](const auto & file) { return json::parse(readFile(file)); },                            \
-            [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
-    }
 
-TEST_JSON(PublicKeyTest, simple, (fetchers::PublicKey{.type = "ssh-rsa", .key = "ABCDE"}))
+TEST_P(PublicKeyTest, to_json)
+{
+    const auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
 
-TEST_JSON(PublicKeyTest, defaultType, fetchers::PublicKey{.key = "ABCDE"})
+INSTANTIATE_TEST_SUITE_P(
-
+    PublicKeyJSON,
-#undef TEST_JSON
+    PublicKeyTest,
+    ::testing::Values(
+        std::pair{
+            "simple",
+            fetchers::PublicKey{
+                .type = "ssh-rsa",
+                .key = "ABCDE",
+            },
+        },
+        std::pair{
+            "defaultType",
+            fetchers::PublicKey{
+                .key = "ABCDE",
+            },
+        }));
 
 TEST_F(PublicKeyTest, PublicKey_noRoundTrip_from_json)
 {

src/libfetchers/fetchers.cc

@@ -3,8 +3,8 @@
 #include "nix/util/source-path.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/util/json-utils.hh"
-#include "nix/fetchers/store-path-accessor.hh"
 #include "nix/fetchers/fetch-settings.hh"
+#include "nix/fetchers/fetch-to-store.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -332,10 +332,20 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
 
             debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
 
-            auto accessor = makeStorePathAccessor(store, storePath);
+            // We just ensured the store object was there
+            auto accessor = ref{store->getFSAccessor(storePath)};
 
             accessor->fingerprint = getFingerprint(store);
 
+            // Store a cache entry for the substituted tree so later fetches
+            // can reuse the existing nar instead of copying the unpacked
+            // input back into the store on every evaluation.
+            if (accessor->fingerprint) {
+                ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
+                auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
+                settings->getCache()->upsert(cacheKey, *store, {}, storePath);
+            }
+
             accessor->setPathDisplay("«" + to_string() + "»");
 
             return {accessor, *this};
@@ -509,7 +519,7 @@ fetchers::PublicKey adl_serializer<fetchers::PublicKey>::from_json(const json &
     return res;
 }
 
-void adl_serializer<fetchers::PublicKey>::to_json(json & json, fetchers::PublicKey p)
+void adl_serializer<fetchers::PublicKey>::to_json(json & json, const fetchers::PublicKey & p)
 {
     json["type"] = p.type;
     json["key"] = p.key;

src/libfetchers/git.cc

@@ -15,6 +15,7 @@
 #include "nix/fetchers/fetch-settings.hh"
 #include "nix/util/json-utils.hh"
 #include "nix/util/archive.hh"
+#include "nix/util/mounted-source-accessor.hh"
 
 #include <regex>
 #include <string.h>

src/libfetchers/github.cc

@@ -398,8 +398,8 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
         Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
-        auto json = nlohmann::json::parse(
+        auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
-            readFile(store->toRealPath(downloadFile(store, *input.settings, url, "source", headers).storePath)));
+        auto json = nlohmann::json::parse(store->getFSAccessor(downloadResult.storePath)->readFile(CanonPath::root));
 
         return RefInfo{
             .rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1),
@@ -472,8 +472,8 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
         Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
-        auto json = nlohmann::json::parse(
+        auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
-            readFile(store->toRealPath(downloadFile(store, *input.settings, url, "source", headers).storePath)));
+        auto json = nlohmann::json::parse(store->getFSAccessor(downloadResult.storePath)->readFile(CanonPath::root));
 
         if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
             return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)};

src/libfetchers/include/nix/fetchers/meson.build

@@ -11,6 +11,5 @@ headers = files(
   'git-utils.hh',
   'input-cache.hh',
   'registry.hh',
-  'store-path-accessor.hh',
   'tarball.hh',
 )

src/libfetchers/include/nix/fetchers/store-path-accessor.hh

@@ -1,14 +0,0 @@
-#pragma once
-
-#include "nix/util/source-path.hh"
-
-namespace nix {
-
-class StorePath;
-class Store;
-
-ref<SourceAccessor> makeStorePathAccessor(ref<Store> store, const StorePath & storePath);
-
-SourcePath getUnfilteredRootPath(CanonPath path);
-
-} // namespace nix

src/libfetchers/mercurial.cc

@@ -6,7 +6,6 @@
 #include "nix/util/tarfile.hh"
 #include "nix/store/store-api.hh"
 #include "nix/util/url-parts.hh"
-#include "nix/fetchers/store-path-accessor.hh"
 #include "nix/fetchers/fetch-settings.hh"
 
 #include <sys/time.h>
@@ -331,7 +330,8 @@ struct MercurialInputScheme : InputScheme
 
         auto storePath = fetchToStore(store, input);
 
-        auto accessor = makeStorePathAccessor(store, storePath);
+        // We just added it, it should be there.
+        auto accessor = ref{store->getFSAccessor(storePath)};
 
         accessor->setPathDisplay("«" + input.to_string() + "»");
 

src/libfetchers/meson.build

@@ -32,6 +32,7 @@ libgit2 = dependency('libgit2', version : '>= 1.9')
 deps_private += libgit2
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'attrs.cc',
@@ -49,7 +50,6 @@ sources = files(
   'mercurial.cc',
   'path.cc',
   'registry.cc',
-  'store-path-accessor.cc',
   'tarball.cc',
 )
 

src/libfetchers/path.cc

@@ -1,7 +1,6 @@
 #include "nix/fetchers/fetchers.hh"
 #include "nix/store/store-api.hh"
 #include "nix/util/archive.hh"
-#include "nix/fetchers/store-path-accessor.hh"
 #include "nix/fetchers/cache.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/fetch-settings.hh"
@@ -153,7 +152,7 @@ struct PathInputScheme : InputScheme
         if (!input.getLastModified())
             input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
 
-        return {makeStorePathAccessor(store, *storePath), std::move(input)};
+        return {ref{store->getFSAccessor(*storePath)}, std::move(input)};
     }
 
     std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override

src/libfetchers/store-path-accessor.cc

@@ -1,11 +0,0 @@
-#include "nix/fetchers/store-path-accessor.hh"
-#include "nix/store/store-api.hh"
-
-namespace nix {
-
-ref<SourceAccessor> makeStorePathAccessor(ref<Store> store, const StorePath & storePath)
-{
-    return projectSubdirSourceAccessor(store->getFSAccessor(), storePath.to_string());
-}
-
-} // namespace nix

src/libfetchers/tarball.cc

@@ -6,7 +6,6 @@
 #include "nix/util/archive.hh"
 #include "nix/util/tarfile.hh"
 #include "nix/util/types.hh"
-#include "nix/fetchers/store-path-accessor.hh"
 #include "nix/store/store-api.hh"
 #include "nix/fetchers/git-utils.hh"
 #include "nix/fetchers/fetch-settings.hh"
@@ -354,7 +353,7 @@ struct FileInputScheme : CurlInputScheme
         auto narHash = store->queryPathInfo(file.storePath)->narHash;
         input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
-        auto accessor = makeStorePathAccessor(store, file.storePath);
+        auto accessor = ref{store->getFSAccessor(file.storePath)};
 
         accessor->setPathDisplay("«" + input.to_string() + "»");
 

src/libflake-c/meson.build

@@ -32,6 +32,7 @@ deps_public_maybe_subproject = [
 subdir('nix-meson-build-support/subprojects')
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'nix_api_flake.cc',

src/libflake-tests/meson.build

@@ -34,6 +34,7 @@ gtest = dependency('gtest', main : true)
 deps_private += gtest
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'flakeref.cc',
@@ -58,7 +59,7 @@ this_exe = executable(
 test(
   meson.project_name(),
   this_exe,
-  env : {
+  env : asan_test_options_env + {
     '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data',
     'NIX_CONFIG' : 'extra-experimental-features = flakes',
     'HOME' : meson.current_build_dir() / 'test-home',

src/libflake-tests/package.nix

@@ -59,6 +59,7 @@ mkMesonExecutable (finalAttrs: {
             buildInputs = [ writableTmpDirAsHomeHook ];
           }
           (''
+            export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0
             export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
             export NIX_CONFIG="extra-experimental-features = flakes"
             ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}

src/libflake/meson.build

@@ -29,6 +29,7 @@ nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 subdir('nix-meson-build-support/generate-header')
 

src/libmain-c/meson.build

@@ -28,6 +28,7 @@ deps_public_maybe_subproject = [
 subdir('nix-meson-build-support/subprojects')
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'nix_api_main.cc',

src/libmain-c/nix_api_main.cc

@@ -4,6 +4,7 @@
 #include "nix_api_util_internal.h"
 
 #include "nix/main/plugin.hh"
+#include "nix/main/loggers.hh"
 
 extern "C" {
 
@@ -17,4 +18,16 @@ nix_err nix_init_plugins(nix_c_context * context)
     NIXC_CATCH_ERRS
 }
 
+nix_err nix_set_log_format(nix_c_context * context, const char * format)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    if (format == nullptr)
+        return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Log format is null");
+    try {
+        nix::setLogFormat(format);
+    }
+    NIXC_CATCH_ERRS
+}
+
 } // extern "C"

src/libmain-c/nix_api_main.h

@@ -30,6 +30,14 @@ extern "C" {
  */
 nix_err nix_init_plugins(nix_c_context * context);
 
+/**
+ * @brief Sets the log format
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] format The string name of the format.
+ */
+nix_err nix_set_log_format(nix_c_context * context, const char * format);
+
 // cffi end
 #ifdef __cplusplus
 }

src/libmain/meson.build

@@ -53,6 +53,7 @@ config_priv_h = configure_file(
 )
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'common-args.cc',

src/libstore-c/meson.build

@@ -26,6 +26,7 @@ deps_public_maybe_subproject = [
 subdir('nix-meson-build-support/subprojects')
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'nix_api_store.cc',

src/libstore-c/nix_api_store.cc

@@ -166,11 +166,44 @@ void nix_store_path_free(StorePath * sp)
     delete sp;
 }
 
+void nix_derivation_free(nix_derivation * drv)
+{
+    delete drv;
+}
+
 StorePath * nix_store_path_clone(const StorePath * p)
 {
     return new StorePath{p->path};
 }
 
+nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store, const char * json)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto drv = static_cast<nix::Derivation>(nlohmann::json::parse(json));
+
+        auto drvPath = nix::writeDerivation(*store->ptr, drv, nix::NoRepair, /* read only */ true);
+
+        drv.checkInvariants(*store->ptr, drvPath);
+
+        return new nix_derivation{drv};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto ret = nix::writeDerivation(*store->ptr, derivation->drv, nix::NoRepair);
+
+        return new StorePath{ret};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
 nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path)
 {
     if (context)

src/libstore-c/nix_api_store.h

@@ -23,6 +23,8 @@ extern "C" {
 typedef struct Store Store;
 /** @brief Nix store path */
 typedef struct StorePath StorePath;
+/** @brief Nix Derivation */
+typedef struct nix_derivation nix_derivation;
 
 /**
  * @brief Initializes the Nix store library
@@ -207,6 +209,32 @@ nix_err nix_store_realise(
 nix_err
 nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data);
 
+/**
+ * @brief Create a `nix_derivation` from a JSON representation of that derivation.
+ *
+ * @param[out] context Optional, stores error information.
+ * @param[in] store nix store reference.
+ * @param[in] json JSON of the derivation as a string.
+ */
+nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store, const char * json);
+
+/**
+ * @brief Add the given `nix_derivation` to the given store
+ *
+ * @param[out] context Optional, stores error information.
+ * @param[in] store nix store reference. The derivation will be inserted here.
+ * @param[in] derivation nix_derivation to insert into the given store.
+ */
+StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation);
+
+/**
+ * @brief Deallocate a `nix_derivation`
+ *
+ * Does not fail.
+ * @param[in] drv the derivation to free
+ */
+void nix_derivation_free(nix_derivation * drv);
+
 /**
  * @brief Copy the closure of `path` from `srcStore` to `dstStore`.
  *

src/libstore-c/nix_api_store_internal.h

@@ -1,6 +1,7 @@
 #ifndef NIX_API_STORE_INTERNAL_H
 #define NIX_API_STORE_INTERNAL_H
 #include "nix/store/store-api.hh"
+#include "nix/store/derivations.hh"
 
 extern "C" {
 
@@ -14,6 +15,11 @@ struct StorePath
     nix::StorePath path;
 };
 
+struct nix_derivation
+{
+    nix::Derivation drv;
+};
+
 } // extern "C"
 
 #endif

src/libstore-test-support/include/nix/store/tests/meson.build

@@ -9,4 +9,5 @@ headers = files(
   'outputs-spec.hh',
   'path.hh',
   'protocol.hh',
+  'test-main.hh',
 )

src/libstore-test-support/include/nix/store/tests/nix_api_store.hh

@@ -12,33 +12,32 @@
 #include <gtest/gtest.h>
 
 namespace nixC {
-class nix_api_store_test : public nix_api_util_context
+
+class nix_api_store_test_base : public nix_api_util_context
 {
 public:
-    nix_api_store_test()
+    nix_api_store_test_base()
     {
         nix_libstore_init(ctx);
-        init_local_store();
     };
 
-    ~nix_api_store_test() override
+    ~nix_api_store_test_base() override
     {
-        nix_store_free(store);
+        if (exists(std::filesystem::path{nixDir})) {
-
             for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) {
                 std::filesystem::permissions(path, std::filesystem::perms::owner_all);
             }
             std::filesystem::remove_all(nixDir);
         }
+    }
 
-    Store * store;
     std::string nixDir;
     std::string nixStoreDir;
     std::string nixStateDir;
     std::string nixLogDir;
 
 protected:
-    void init_local_store()
+    Store * open_local_store()
     {
 #ifdef _WIN32
         // no `mkdtemp` with MinGW
@@ -66,11 +65,37 @@ protected:
 
         const char ** params[] = {p1, p2, p3, nullptr};
 
-        store = nix_store_open(ctx, "local", params);
+        auto * store = nix_store_open(ctx, "local", params);
         if (!store) {
             std::string errMsg = nix_err_msg(nullptr, ctx, nullptr);
-            ASSERT_NE(store, nullptr) << "Could not open store: " << errMsg;
+            EXPECT_NE(store, nullptr) << "Could not open store: " << errMsg;
+            assert(store);
         };
+        return store;
     }
 };
+
+class nix_api_store_test : public nix_api_store_test_base
+{
+public:
+    nix_api_store_test()
+        : nix_api_store_test_base{}
+    {
+        init_local_store();
+    };
+
+    ~nix_api_store_test() override
+    {
+        nix_store_free(store);
+    }
+
+    Store * store;
+
+protected:
+    void init_local_store()
+    {
+        store = open_local_store();
+    }
+};
+
 } // namespace nixC

src/libstore-test-support/include/nix/store/tests/test-main.hh

@@ -0,0 +1,13 @@
+#pragma once
+
+///@file
+
+namespace nix {
+
+/**
+ * Call this for a GTest test suite that will including performing Nix
+ * builds, before running tests.
+ */
+int testMainForBuidingPre(int argc, char ** argv);
+
+} // namespace nix

src/libstore-test-support/meson.build

@@ -29,11 +29,13 @@ rapidcheck = dependency('rapidcheck')
 deps_public += rapidcheck
 
 subdir('nix-meson-build-support/common')
+subdir('nix-meson-build-support/asan-options')
 
 sources = files(
   'derived-path.cc',
   'outputs-spec.cc',
   'path.cc',
+  'test-main.cc',
 )
 
 subdir('include/nix/store/tests')

src/libstore-test-support/test-main.cc

@@ -0,0 +1,47 @@
+#include <cstdlib>
+
+#include "nix/store/globals.hh"
+#include "nix/util/logging.hh"
+
+#include "nix/store/tests/test-main.hh"
+
+namespace nix {
+
+int testMainForBuidingPre(int argc, char ** argv)
+{
+    if (argc > 1 && std::string_view(argv[1]) == "__build-remote") {
+        printError("test-build-remote: not supported in libexpr unit tests");
+        return EXIT_FAILURE;
+    }
+
+    // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook.
+    settings.buildHook = {};
+
+    // No substituters, unless a test specifically requests.
+    settings.substituters = {};
+
+#ifdef __linux__ // should match the conditional around sandboxBuildDir declaration.
+
+    // When building and testing nix within the host's Nix sandbox, our store dir will be located in the host's
+    // sandboxBuildDir, e.g.: Host
+    //   storeDir = /nix/store
+    //   sandboxBuildDir = /build
+    // This process
+    //   storeDir = /build/foo/bar/store
+    //   sandboxBuildDir = /build
+    // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different
+    // sandboxBuildDir.
+    settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
+#endif
+
+#ifdef __APPLE__
+    // Avoid this error, when already running in a sandbox:
+    // sandbox-exec: sandbox_apply: Operation not permitted
+    settings.sandboxMode = smDisabled;
+    setEnv("_NIX_TEST_NO_SANDBOX", "1");
+#endif
+
+    return EXIT_SUCCESS;
+}
+
+} // namespace nix

src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json

@@ -21,5 +21,6 @@
       "method": "nar"
     }
   },
-  "system": "my-system"
+  "system": "my-system",
+  "version": 3
 }

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json

@@ -32,5 +32,6 @@
     ],
     "system": "my-system"
   },
-  "system": "my-system"
+  "system": "my-system",
+  "version": 3
 }

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json

@@ -10,14 +10,14 @@
     "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
   },
   "inputDrvs": {
-    "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
+    "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
       "dynamicOutputs": {},
       "outputs": [
         "dev",
         "out"
       ]
     },
-    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
+    "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
       "dynamicOutputs": {},
       "outputs": [
         "dev",
@@ -26,7 +26,7 @@
     }
   },
   "inputSrcs": [
-    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+    "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
   ],
   "name": "advanced-attributes-structured-attrs",
   "outputs": {
@@ -100,5 +100,6 @@
     ],
     "system": "my-system"
   },
-  "system": "my-system"
+  "system": "my-system",
+  "version": 3
 }

src/libstore-tests/data/derivation/ca/advanced-attributes.json

@@ -26,14 +26,14 @@
     "system": "my-system"
   },
   "inputDrvs": {
-    "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
+    "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
       "dynamicOutputs": {},
       "outputs": [
         "dev",
         "out"
       ]
     },
-    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
+    "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
       "dynamicOutputs": {},
       "outputs": [
         "dev",
@@ -42,7 +42,7 @@
     }
   },
   "inputSrcs": [
-    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+    "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
   ],
   "name": "advanced-attributes",
   "outputs": {
@@ -51,5 +51,6 @@
       "method": "nar"
     }
   },
-  "system": "my-system"
+  "system": "my-system",
+  "version": 3
 }

src/libstore-tests/data/derivation/ca/self-contained.json

@@ -0,0 +1,24 @@
+{
+  "args": [
+    "-c",
+    "echo $name foo > $out"
+  ],
+  "builder": "/bin/sh",
+  "env": {
+    "builder": "/bin/sh",
+    "name": "myname",
+    "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9",
+    "system": "x86_64-linux"
+  },
+  "inputDrvs": {},
+  "inputSrcs": [],
+  "name": "myname",
+  "outputs": {
+    "out": {
+      "hashAlgo": "sha256",
+      "method": "nar"
+    }
+  },
+  "system": "x86_64-linux",
+  "version": 3
+}

src/libstore-tests/data/derivation/dyn-dep-derivation.json

@@ -8,7 +8,7 @@
     "BIG_BAD": "WOLF"
   },
   "inputDrvs": {
-    "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
+    "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
       "dynamicOutputs": {
         "cat": {
           "dynamicOutputs": {},
@@ -30,9 +30,10 @@
     }
   },
   "inputSrcs": [
-    "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
+    "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
   ],
   "name": "dyn-dep-derivation",
   "outputs": {},
-  "system": "wasm-sel4"
+  "system": "wasm-sel4",
+  "version": 3
 }