From c4813b8cbc3f8559c60d1a387036c64e2649734c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 May 2025 14:48:51 +0200 Subject: [PATCH 01/73] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index f01356823..69886179f 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.29.0 +2.29.1 From acc3cd460d8968b6a90bf8f9076280b4aa7a6961 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 23 May 2025 23:33:59 +0200 Subject: [PATCH 02/73] Don't use 'callback' object that we may have moved out of (cherry picked from commit fa6e10ea6a87127ae813a708ccc97e708982f93f) --- src/libstore/http-binary-cache-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 2b591dda9..e44d146b9 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -176,13 +176,13 @@ protected: void getFile(const std::string & path, Callback> callback) noexcept override { + auto callbackPtr = std::make_shared(std::move(callback)); + try { checkEnabled(); auto request(makeRequest(path)); - auto callbackPtr = std::make_shared(std::move(callback)); - getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future result) { try { @@ -198,7 +198,7 @@ protected: }}); } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); return; } } From 8ca4d2ef08e8e19c893ec186cfe8aeee91e94041 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 24 May 2025 00:14:32 +0200 Subject: [PATCH 03/73] fromStructuredAttrs(): Don't crash if exportReferencesGraph is a string Fixes error: [json.exception.type_error.302] type must be array, but is string and other crashes. Fixes #13254. (cherry picked from commit d877b0c0cc4795d17d10b9b9039f2de828152c55) --- src/libstore/derivation-options.cc | 9 +++++++-- src/libstore/misc.cc | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index e031f8447..f6bac2868 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -211,8 +211,13 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); if (!e || !e->is_object()) return ret; - for (auto & [key, storePathsJson] : getObject(*e)) { - ret.insert_or_assign(key, storePathsJson); + for (auto & [key, value] : getObject(*e)) { + if (value.is_array()) + ret.insert_or_assign(key, value); + else if (value.is_string()) + ret.insert_or_assign(key, StringSet{value}); + else + throw Error("'exportReferencesGraph' value is not an array or a string"); } } else { auto s = getOr(env, "exportReferencesGraph", ""); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 967c91d72..dabae647f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -225,6 +225,8 @@ void Store::queryMissing(const std::vector & targets, auto parsedDrv = StructuredAttrs::tryParse(drv->env); DerivationOptions drvOptions; try { + // FIXME: this is a lot of work just to get the value + // of `allowSubstitutes`. drvOptions = DerivationOptions::fromStructuredAttrs( drv->env, parsedDrv ? &*parsedDrv : nullptr); From d068b82c2f27d4105f54b43dde34a67dbd7d6db9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 24 May 2025 00:40:06 +0200 Subject: [PATCH 04/73] Add test (cherry picked from commit c66eb9cef77c3462d0324b258d0c5e0b8e4f4e7f) --- tests/functional/structured-attrs-shell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/structured-attrs-shell.nix b/tests/functional/structured-attrs-shell.nix index a819e39cd..e9b9f1e39 100644 --- a/tests/functional/structured-attrs-shell.nix +++ b/tests/functional/structured-attrs-shell.nix @@ -21,7 +21,7 @@ mkDerivation { "b" "c" ]; - exportReferencesGraph.refs = [ dep ]; + exportReferencesGraph.refs = dep; buildCommand = '' touch ''${outputs[out]}; touch ''${outputs[dev]} ''; From cfba4b3bf41ed01a30a98e6bc5db96c909d2e73d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2025 23:30:16 +0200 Subject: [PATCH 05/73] Drop magic-nix-cache This no longer works, see https://determinate.systems/posts/magic-nix-cache-free-tier-eol/. (cherry picked from commit 9cc8be26747a0206613421a1ba1c3b1f54212e8b) --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29cb33f56..fb70fae87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,6 @@ jobs: extra_nix_config: | sandbox = true max-jobs = 1 - - uses: DeterminateSystems/magic-nix-cache-action@main # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 @@ -134,7 +133,6 @@ jobs: - uses: cachix/install-nix-action@v31 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz @@ -176,7 +174,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - run: | nix build -L \ .#hydraJobs.tests.functional_user \ @@ -202,5 +199,4 @@ jobs: repository: NixOS/flake-regressions-data path: flake-regressions/tests - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh From a2567f6d7ae9bcd7771a8790c0a9196e90ce097d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 29 May 2025 19:35:12 +0000 Subject: [PATCH 06/73] Clear `displayPrefix` in `makeEmptySourceAccessor` Judging by the comment for `makeEmptySourceAccessor` the prefix has to be empty: > Return a source accessor that contains only an empty root directory. Fixes #13295. (cherry picked from commit fba1bb0c137036adc5127afe4183f45ab3dde61d) --- src/libutil/memory-source-accessor.cc | 4 ++++ tests/functional/pure-eval.sh | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index 7764ff946..5612c9454 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -187,6 +187,10 @@ void MemorySink::createSymlink(const CanonPath & path, const std::string & targe ref makeEmptySourceAccessor() { static auto empty = make_ref().cast(); + /* Don't forget to clear the display prefix, as the default constructed + SourceAccessor has the «unknown» prefix. Since this accessor is supposed + to mimic an empty root directory the prefix needs to be empty. */ + empty->setPathDisplay(""); return empty; } diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh index 250381099..45a65f9ab 100755 --- a/tests/functional/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -34,3 +34,15 @@ rm -rf $TEST_ROOT/eval-out (! nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ "." = "bla"; }') (! nix eval --expr '~/foo') + +expectStderr 0 nix eval --expr "/some/absolute/path" \ + | grepQuiet "/some/absolute/path" + +expectStderr 0 nix eval --expr "/some/absolute/path" --impure \ + | grepQuiet "/some/absolute/path" + +expectStderr 0 nix eval --expr "some/relative/path" \ + | grepQuiet "$PWD/some/relative/path" + +expectStderr 0 nix eval --expr "some/relative/path" --impure \ + | grepQuiet "$PWD/some/relative/path" From fcdffffa37ab99b15490bb633698ee9fe03e7056 Mon Sep 17 00:00:00 2001 From: Seth Flynn Date: Tue, 27 May 2025 22:20:53 -0400 Subject: [PATCH 07/73] lockFlake(): Allow registry lookups for overridden inputs Fixes #13144 (cherry picked from commit d0a23238294198f6702e13d117f75af89dbeac62) --- src/libflake/flake.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 987c9f610..d5dafff7a 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -570,7 +570,7 @@ LockedFlake lockFlake( /* Get the input flake, resolve 'path:./...' flakerefs relative to the parent flake. */ - auto getInputFlake = [&](const FlakeRef & ref) + auto getInputFlake = [&](const FlakeRef & ref, const fetchers::UseRegistries useRegistries) { if (auto resolvedPath = resolveRelativePath()) { return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); @@ -578,7 +578,7 @@ LockedFlake lockFlake( return getFlake( state, ref, - useRegistriesInputs, + useRegistries, inputAttrPath); } }; @@ -660,7 +660,7 @@ LockedFlake lockFlake( } if (mustRefetch) { - auto inputFlake = getInputFlake(oldLock->lockedRef); + auto inputFlake = getInputFlake(oldLock->lockedRef, useRegistriesInputs); nodePaths.emplace(childNode, inputFlake.path.parent()); computeLocks(inputFlake.inputs, childNode, inputAttrPath, oldLock, followsPrefix, inputFlake.path, false); @@ -685,10 +685,11 @@ LockedFlake lockFlake( nuked the next time we update the lock file. That is, overrides are sticky unless you use --no-write-lock-file. */ - auto ref = (input2.ref && explicitCliOverrides.contains(inputAttrPath)) ? *input2.ref : *input.ref; + auto inputIsOverride = explicitCliOverrides.contains(inputAttrPath); + auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; if (input.isFlake) { - auto inputFlake = getInputFlake(*input.ref); + auto inputFlake = getInputFlake(*input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); auto childNode = make_ref( inputFlake.lockedRef, From db3cecdbe04b34b7fd12d11e4b9014d323288a1c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:02:05 +0000 Subject: [PATCH 08/73] Prepare release v3.6.3 From c36fc11bc6655d448d35f2ae0019521b684b2d71 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:02:08 +0000 Subject: [PATCH 09/73] Set .version-determinate to 3.6.3 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index b72762837..4a788a01d 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.2 +3.6.3 From f4a3894ec92c3f4a82d2aecd288a65f4666ec011 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:02:13 +0000 Subject: [PATCH 10/73] Generare release notes for 3.6.3 --- doc/manual/source/SUMMARY.md.in | 1 + .../release-notes-determinate/changes.md | 33 +++++++++++++++++-- .../release-notes-determinate/rl-3.6.3.md | 22 +++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.3.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 8efc01612..3bc62b9c1 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.3 (2025-06-12)](release-notes-determinate/rl-3.6.3.md) - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) - [Release 3.6.0 (2025-05-22)](release-notes-determinate/rl-3.6.0.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index f4ea70747..2aed26069 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.2. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.3. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -44,4 +44,33 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * nix profile: Replace ε and ∅ with descriptive English words by @grahamc in [DeterminateSystems/nix-src#81](https://github.com/DeterminateSystems/nix-src/pull/81) -* Call out that `--keep-failed` with remote builders will keep the failed build directory on that builder by @cole-h in [DeterminateSystems/nix-src#85](https://github.com/DeterminateSystems/nix-src/pull/85) \ No newline at end of file +* Call out that `--keep-failed` with remote builders will keep the failed build directory on that builder by @cole-h in [DeterminateSystems/nix-src#85](https://github.com/DeterminateSystems/nix-src/pull/85) + + +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) + +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) + +* Use 'published' release type to avoid double uploads by @gustavderdrache in [DeterminateSystems/nix-src#90](https://github.com/DeterminateSystems/nix-src/pull/90) + +* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) + +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) + +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) + +* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) + +* Fix: Make the S3 test more robust by @gustavderdrache in [DeterminateSystems/nix-src#101](https://github.com/DeterminateSystems/nix-src/pull/101) + +* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) + +* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) + +* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) + +* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) + +* Go back to x86 native macOS builds by @grahamc in [DeterminateSystems/nix-src#104](https://github.com/DeterminateSystems/nix-src/pull/104) + +* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) \ No newline at end of file diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.3.md b/doc/manual/source/release-notes-determinate/rl-3.6.3.md new file mode 100644 index 000000000..7f0068e6a --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.3.md @@ -0,0 +1,22 @@ +# Release 3.6.3 (2025-06-12) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) +* Use 'published' release type to avoid double uploads by @gustavderdrache in [DeterminateSystems/nix-src#90](https://github.com/DeterminateSystems/nix-src/pull/90) +* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) +* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) +* Fix: Make the S3 test more robust by @gustavderdrache in [DeterminateSystems/nix-src#101](https://github.com/DeterminateSystems/nix-src/pull/101) +* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) +* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) +* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) +* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) +* Go back to x86 native macOS builds by @grahamc in [DeterminateSystems/nix-src#104](https://github.com/DeterminateSystems/nix-src/pull/104) +* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + + +**Full Changelog**: [v3.6.2...v3.6.3](https://github.com/DeterminateSystems/nix-src/compare/v3.6.2...v3.6.3) From 92d90e2aebd75648afaf379750969dd882d59a62 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Jun 2025 22:23:28 +0200 Subject: [PATCH 11/73] Run the Nix test suite with lazy trees enabled --- flake.nix | 6 ++++++ tests/functional/common/init.sh | 1 + tests/functional/package.nix | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/flake.nix b/flake.nix index b5fa93e58..451068f5d 100644 --- a/flake.nix +++ b/flake.nix @@ -220,6 +220,12 @@ ''; repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; + lazyTrees = + nixpkgsFor.${system}.native.nixComponents2.nix-functional-tests.override { + pname = "nix-lazy-trees-tests"; + lazyTrees = true; + }; + /** Checks for our packaging expressions. This shouldn't build anything significant; just check that things diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index 6e9bffec5..7f28a09d7 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -54,6 +54,7 @@ flake-registry = $TEST_ROOT/registry.json show-trace = true include nix.conf.extra trusted-users = $(whoami) +${_NIX_TEST_EXTRA_CONFIG:-} EOF cat > "$NIX_CONF_DIR"/nix.conf.extra < Date: Wed, 11 Jun 2025 22:30:57 +0200 Subject: [PATCH 12/73] Fix flakes test with lazy trees enabled --- tests/functional/flakes/flakes.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 0a52ba08c..e335fe6f3 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -69,7 +69,9 @@ nix flake metadata "$flake1Dir" | grepQuiet 'URL:.*flake1.*' # Test 'nix flake metadata --json'. json=$(nix flake metadata flake1 --json | jq .) [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] -[[ -d $(echo "$json" | jq -r .path) ]] +if [[ $(nix config show lazy-trees) = false ]]; then + [[ -d $(echo "$json" | jq -r .path) ]] +fi [[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) [[ -n $(echo "$json" | jq -r .fingerprint) ]] @@ -161,7 +163,11 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] -[[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +if [[ $(nix config show lazy-trees) = false ]]; then + [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +else + [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +fi # Rerunning the build should not change the lockfile. nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" From b067e6566f97c3b01f7321a19c317e6909a5380c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2025 15:54:34 +0200 Subject: [PATCH 13/73] Git fetcher: Do not consider a null revision (i.e. workdir) to be locked --- src/libfetchers/git.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7730e0db4..2825b72ab 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -882,7 +882,8 @@ struct GitInputScheme : InputScheme bool isLocked(const Input & input) const override { - return (bool) input.getRev(); + auto rev = input.getRev(); + return rev && rev != nullRev; } }; From fca291afc358e4f1c9565dd236db1d0cc87fef24 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2025 16:00:29 +0200 Subject: [PATCH 14/73] Fix NAR hash checking for fetchGit with lazy tees If a NAR hash is specified, we should probably check it. Unfortunately, for now this has the side effect of forcing NAR hash checking of any input that has a NAR hash. --- src/libexpr/paths.cc | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 40c0a23b6..d85f00470 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -77,25 +77,28 @@ StorePath EvalState::mountInput( allowPath(storePath); // FIXME: should just whitelist the entire virtual store + std::optional _narHash; + + auto getNarHash = [&]() + { + if (!_narHash) + // FIXME: use fetchToStore to make it cache this + _narHash = accessor->hashPath(CanonPath::root); + return _narHash; + }; + storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); - if (requireLockable && (!settings.lazyTrees || !input.isLocked()) && !input.getNarHash()) { - // FIXME: use fetchToStore to make it cache this - auto narHash = accessor->hashPath(CanonPath::root); - input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - } + if (requireLockable && (!settings.lazyTrees || !input.isLocked()) && !input.getNarHash()) + input.attrs.insert_or_assign("narHash", getNarHash()->to_string(HashFormat::SRI, true)); - // FIXME: what to do with the NAR hash in lazy mode? - if (!settings.lazyTrees && originalInput.getNarHash()) { - auto expected = originalInput.computeStorePath(*store); - if (storePath != expected) - throw Error( - (unsigned int) 102, - "NAR hash mismatch in input '%s', expected '%s' but got '%s'", - originalInput.to_string(), - store->printStorePath(storePath), - store->printStorePath(expected)); - } + if (originalInput.getNarHash() && *getNarHash() != *originalInput.getNarHash()) + throw Error( + (unsigned int) 102, + "NAR hash mismatch in input '%s', expected '%s' but got '%s'", + originalInput.to_string(), + getNarHash()->to_string(HashFormat::SRI, true), + originalInput.getNarHash()->to_string(HashFormat::SRI, true)); return storePath; } From f6b612135d2593d5a31e5bfd33347591c1f95eb1 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 12 Jun 2025 10:08:42 -0400 Subject: [PATCH 15/73] Update doc/manual/source/release-notes-determinate/changes.md --- doc/manual/source/release-notes-determinate/changes.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 2aed26069..a5b9383e3 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -67,10 +67,4 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) -* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) - -* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) - -* Go back to x86 native macOS builds by @grahamc in [DeterminateSystems/nix-src#104](https://github.com/DeterminateSystems/nix-src/pull/104) - * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) \ No newline at end of file From 4921297695527b82570612721eb8734804243431 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 12 Jun 2025 10:08:59 -0400 Subject: [PATCH 16/73] Update doc/manual/source/release-notes-determinate/rl-3.6.3.md --- doc/manual/source/release-notes-determinate/rl-3.6.3.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.3.md b/doc/manual/source/release-notes-determinate/rl-3.6.3.md index 7f0068e6a..07d7ccaa5 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.3.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.3.md @@ -5,7 +5,6 @@ ## What's Changed * When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) * Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) -* Use 'published' release type to avoid double uploads by @gustavderdrache in [DeterminateSystems/nix-src#90](https://github.com/DeterminateSystems/nix-src/pull/90) * Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) * Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) * Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) From 829d3f85438d7ba1b6e48151a87ea1f7ec2e2295 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 12 Jun 2025 10:09:10 -0400 Subject: [PATCH 17/73] Apply suggestions from code review --- doc/manual/source/release-notes-determinate/rl-3.6.3.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.3.md b/doc/manual/source/release-notes-determinate/rl-3.6.3.md index 07d7ccaa5..b24037b74 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.3.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.3.md @@ -9,12 +9,10 @@ * Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) * Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) * Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) -* Fix: Make the S3 test more robust by @gustavderdrache in [DeterminateSystems/nix-src#101](https://github.com/DeterminateSystems/nix-src/pull/101) * Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) * Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) * Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) -* Go back to x86 native macOS builds by @grahamc in [DeterminateSystems/nix-src#104](https://github.com/DeterminateSystems/nix-src/pull/104) * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) From 6477d7c2ca3ff84f9d1b502d4d62bddfe3b77b0f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2025 16:11:54 +0200 Subject: [PATCH 18/73] mountInput(): Optimize getting the NAR hash for real store paths --- src/libexpr/paths.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index d85f00470..bdf817e3b 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -81,9 +81,13 @@ StorePath EvalState::mountInput( auto getNarHash = [&]() { - if (!_narHash) - // FIXME: use fetchToStore to make it cache this - _narHash = accessor->hashPath(CanonPath::root); + if (!_narHash) { + if (store->isValidPath(storePath)) + _narHash = store->queryPathInfo(storePath)->narHash; + else + // FIXME: use fetchToStore to make it cache this + _narHash = accessor->hashPath(CanonPath::root); + } return _narHash; }; From 279a6b18dba633a63afd69c8917833845db0c204 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2025 16:13:28 +0200 Subject: [PATCH 19/73] Formatting --- flake.nix | 9 ++++----- src/libexpr/paths.cc | 3 +-- tests/functional/package.nix | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index 451068f5d..0207134cd 100644 --- a/flake.nix +++ b/flake.nix @@ -220,11 +220,10 @@ ''; repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; - lazyTrees = - nixpkgsFor.${system}.native.nixComponents2.nix-functional-tests.override { - pname = "nix-lazy-trees-tests"; - lazyTrees = true; - }; + lazyTrees = nixpkgsFor.${system}.native.nixComponents2.nix-functional-tests.override { + pname = "nix-lazy-trees-tests"; + lazyTrees = true; + }; /** Checks for our packaging expressions. diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index bdf817e3b..65b8212e1 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -79,8 +79,7 @@ StorePath EvalState::mountInput( std::optional _narHash; - auto getNarHash = [&]() - { + auto getNarHash = [&]() { if (!_narHash) { if (store->isValidPath(storePath)) _narHash = store->queryPathInfo(storePath)->narHash; diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 3185cdf9a..799026ebe 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -28,7 +28,7 @@ test-daemon ? null, # Whether to run tests with lazy trees enabled. - lazyTrees ? false + lazyTrees ? false, }: let From e565571f694d4e2c87ecef0b41a01e5eae67e55e Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 12 Jun 2025 08:25:10 -0700 Subject: [PATCH 20/73] Re-supply inadvertently deleted word --- src/libmain/plugin.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc index f5eddabdd..760a096ad 100644 --- a/src/libmain/plugin.cc +++ b/src/libmain/plugin.cc @@ -60,7 +60,7 @@ struct PluginSettings : Config itself, they must be DSOs compatible with the instance of Nix running at the time (i.e. compiled against the same headers, not linked to any incompatible libraries). They should not be linked to - any Nix libraries directly, as those are already at load + any Nix libraries directly, as those are already available at load time. If an entry in the list is a directory, all files in the directory From b8e8214d5f706dec6189b4d31385ab5a5873a8cc Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 12 Jun 2025 08:28:01 -0700 Subject: [PATCH 21/73] Fix one more 'will' --- src/libstore/include/nix/store/filetransfer.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 259af7640..745aeb29e 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -46,7 +46,7 @@ struct FileTransferSettings : Config )"}; Setting tries{this, 5, "download-attempts", - "The number of times Nix will attempt to download a file before giving up."}; + "The number of times Nix attempts to download a file before giving up."}; Setting downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size", R"( From 642d1bdfb2ba55a475784c801733117acbcc2c02 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 12 Jun 2025 13:23:39 -0400 Subject: [PATCH 22/73] Update doc/manual/source/release-notes-determinate/rl-3.6.3.md --- doc/manual/source/release-notes-determinate/rl-3.6.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.3.md b/doc/manual/source/release-notes-determinate/rl-3.6.3.md index b24037b74..1989a8c8e 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.3.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.3.md @@ -13,6 +13,7 @@ * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) * Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) * Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) +* Run the Nix test suite against lazy trees by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) From 760e6e5f1e561c3b105a17aefcfba7efbf168d1c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Jun 2025 23:23:24 +0000 Subject: [PATCH 23/73] Prepare release v3.6.4 From 17de8fd29c4cd8f3673d159582be7efbdcf50b87 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Jun 2025 23:23:27 +0000 Subject: [PATCH 24/73] Set .version-determinate to 3.6.4 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 4a788a01d..0f44168a4 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.3 +3.6.4 From ac5cbe7c888ab6f63f91a3a9f3fa22fb763d81c9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Jun 2025 23:23:32 +0000 Subject: [PATCH 25/73] Generare release notes for 3.6.4 --- doc/manual/source/SUMMARY.md.in | 1 + .../release-notes-determinate/changes.md | 39 ++++++++++++++++++- .../release-notes-determinate/rl-3.6.4.md | 25 ++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.4.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 3bc62b9c1..efee2213d 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.4 (2025-06-12)](release-notes-determinate/rl-3.6.4.md) - [Release 3.6.3 (2025-06-12)](release-notes-determinate/rl-3.6.3.md) - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index a5b9383e3..9a08e1631 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.3. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.4. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -67,4 +67,39 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) -* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) \ No newline at end of file +* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + + +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) + +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) + +* Use 'published' release type to avoid double uploads by @gustavderdrache in [DeterminateSystems/nix-src#90](https://github.com/DeterminateSystems/nix-src/pull/90) + +* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) + +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) + +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) + +* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) + +* Fix: Make the S3 test more robust by @gustavderdrache in [DeterminateSystems/nix-src#101](https://github.com/DeterminateSystems/nix-src/pull/101) + +* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) + +* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) + +* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) + +* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) + +* Go back to x86 native macOS builds by @grahamc in [DeterminateSystems/nix-src#104](https://github.com/DeterminateSystems/nix-src/pull/104) + +* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + +* Run the Nix test suite with lazy trees enabled by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) + +* Re-supply inadvertently deleted word by @lucperkins in [DeterminateSystems/nix-src#107](https://github.com/DeterminateSystems/nix-src/pull/107) + +* Release v3.6.3 by @github-actions in [DeterminateSystems/nix-src#106](https://github.com/DeterminateSystems/nix-src/pull/106) \ No newline at end of file diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.4.md b/doc/manual/source/release-notes-determinate/rl-3.6.4.md new file mode 100644 index 000000000..5cd4d28e4 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.4.md @@ -0,0 +1,25 @@ +# Release 3.6.4 (2025-06-12) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) +* Use 'published' release type to avoid double uploads by @gustavderdrache in [DeterminateSystems/nix-src#90](https://github.com/DeterminateSystems/nix-src/pull/90) +* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) +* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) +* Fix: Make the S3 test more robust by @gustavderdrache in [DeterminateSystems/nix-src#101](https://github.com/DeterminateSystems/nix-src/pull/101) +* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) +* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) +* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) +* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) +* Go back to x86 native macOS builds by @grahamc in [DeterminateSystems/nix-src#104](https://github.com/DeterminateSystems/nix-src/pull/104) +* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) +* Run the Nix test suite with lazy trees enabled by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) +* Re-supply inadvertently deleted word by @lucperkins in [DeterminateSystems/nix-src#107](https://github.com/DeterminateSystems/nix-src/pull/107) +* Release v3.6.3 by @github-actions in [DeterminateSystems/nix-src#106](https://github.com/DeterminateSystems/nix-src/pull/106) + + +**Full Changelog**: [v3.6.2...v3.6.4](https://github.com/DeterminateSystems/nix-src/compare/v3.6.2...v3.6.4) From da67f596de63afc567bdeb14891ba68ab3b990ac Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 12 Jun 2025 19:51:56 -0400 Subject: [PATCH 26/73] Mark 3.6.3 as revoked, and use its notes for 3.6.4 --- doc/manual/source/SUMMARY.md.in | 2 +- .../release-notes-determinate/changes.md | 36 ++----------------- .../release-notes-determinate/rl-3.6.3.md | 20 ----------- .../release-notes-determinate/rl-3.6.4.md | 7 +--- 4 files changed, 4 insertions(+), 61 deletions(-) delete mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.3.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index efee2213d..24d6a9cd5 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -130,7 +130,7 @@ - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) - [Release 3.6.4 (2025-06-12)](release-notes-determinate/rl-3.6.4.md) - - [Release 3.6.3 (2025-06-12)](release-notes-determinate/rl-3.6.3.md) + - ~~Release 3.6.3 (2025-06-12) (revoked)~~ - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) - [Release 3.6.0 (2025-05-22)](release-notes-determinate/rl-3.6.0.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 9a08e1631..37ff16592 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -45,31 +45,11 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * nix profile: Replace ε and ∅ with descriptive English words by @grahamc in [DeterminateSystems/nix-src#81](https://github.com/DeterminateSystems/nix-src/pull/81) * Call out that `--keep-failed` with remote builders will keep the failed build directory on that builder by @cole-h in [DeterminateSystems/nix-src#85](https://github.com/DeterminateSystems/nix-src/pull/85) - + -* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) - -* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) - -* Use 'published' release type to avoid double uploads by @gustavderdrache in [DeterminateSystems/nix-src#90](https://github.com/DeterminateSystems/nix-src/pull/90) - -* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) - -* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) - -* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) - -* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) - -* Fix: Make the S3 test more robust by @gustavderdrache in [DeterminateSystems/nix-src#101](https://github.com/DeterminateSystems/nix-src/pull/101) - -* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) - -* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) - -* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + * When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) * Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) @@ -90,16 +70,4 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) -* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) - -* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) - -* Go back to x86 native macOS builds by @grahamc in [DeterminateSystems/nix-src#104](https://github.com/DeterminateSystems/nix-src/pull/104) - * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) - -* Run the Nix test suite with lazy trees enabled by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) - -* Re-supply inadvertently deleted word by @lucperkins in [DeterminateSystems/nix-src#107](https://github.com/DeterminateSystems/nix-src/pull/107) - -* Release v3.6.3 by @github-actions in [DeterminateSystems/nix-src#106](https://github.com/DeterminateSystems/nix-src/pull/106) \ No newline at end of file diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.3.md b/doc/manual/source/release-notes-determinate/rl-3.6.3.md deleted file mode 100644 index 1989a8c8e..000000000 --- a/doc/manual/source/release-notes-determinate/rl-3.6.3.md +++ /dev/null @@ -1,20 +0,0 @@ -# Release 3.6.3 (2025-06-12) - -* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). - -## What's Changed -* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) -* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) -* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) -* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) -* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) -* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) -* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) -* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) -* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) -* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) -* Run the Nix test suite against lazy trees by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) -* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) - - -**Full Changelog**: [v3.6.2...v3.6.3](https://github.com/DeterminateSystems/nix-src/compare/v3.6.2...v3.6.3) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.4.md b/doc/manual/source/release-notes-determinate/rl-3.6.4.md index 5cd4d28e4..64086bfba 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.4.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.4.md @@ -5,21 +5,16 @@ ## What's Changed * When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) * Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) -* Use 'published' release type to avoid double uploads by @gustavderdrache in [DeterminateSystems/nix-src#90](https://github.com/DeterminateSystems/nix-src/pull/90) * Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) * Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) * Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) * Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) -* Fix: Make the S3 test more robust by @gustavderdrache in [DeterminateSystems/nix-src#101](https://github.com/DeterminateSystems/nix-src/pull/101) * Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) * Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) * Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) -* Go back to x86 native macOS builds by @grahamc in [DeterminateSystems/nix-src#104](https://github.com/DeterminateSystems/nix-src/pull/104) +* Run the Nix test suite against lazy trees by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) -* Run the Nix test suite with lazy trees enabled by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) -* Re-supply inadvertently deleted word by @lucperkins in [DeterminateSystems/nix-src#107](https://github.com/DeterminateSystems/nix-src/pull/107) -* Release v3.6.3 by @github-actions in [DeterminateSystems/nix-src#106](https://github.com/DeterminateSystems/nix-src/pull/106) **Full Changelog**: [v3.6.2...v3.6.4](https://github.com/DeterminateSystems/nix-src/compare/v3.6.2...v3.6.4) From 37ab15a9f399d482904cdd90994539c61de0b356 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 13 Jun 2025 09:29:42 -0400 Subject: [PATCH 27/73] Drop the complainy 3.6.3 line --- doc/manual/source/SUMMARY.md.in | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 24d6a9cd5..781dba88c 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -130,7 +130,6 @@ - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) - [Release 3.6.4 (2025-06-12)](release-notes-determinate/rl-3.6.4.md) - - ~~Release 3.6.3 (2025-06-12) (revoked)~~ - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) - [Release 3.6.0 (2025-05-22)](release-notes-determinate/rl-3.6.0.md) From 802f58540618ced207c31822af8feb7f2e67853b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Jun 2025 16:56:13 +0200 Subject: [PATCH 28/73] Add lazy-locks setting This determines whether lock file entries omit a NAR hash. They're included by default to make lazy trees compatible with older clients. --- src/libexpr/include/nix/expr/eval-settings.hh | 13 +++++++++++++ src/libexpr/paths.cc | 2 +- tests/functional/flakes/flakes.sh | 9 +++++---- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index 782f5f9e1..cb4726837 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -262,6 +262,19 @@ struct EvalSettings : Config R"( If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily. )"}; + + // FIXME: this setting should really be in libflake, but it's + // currently needed in mountInput(). + Setting lazyLocks{ + this, + false, + "lazy-locks", + R"( + If enabled, Nix will only include NAR hashes in lock file entries if they're necessary to lock the input (i.e. when there is no other attribute that allows the content to be verified, like a Git revision). + This is not backward compatible with older versions of Nix. + If disabled, lock file entries will always contain a NAR hash. + )" + }; }; /** diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 65b8212e1..7bac317d9 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -92,7 +92,7 @@ StorePath EvalState::mountInput( storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); - if (requireLockable && (!settings.lazyTrees || !input.isLocked()) && !input.getNarHash()) + if (requireLockable && (!settings.lazyTrees || !settings.lazyLocks || !input.isLocked()) && !input.getNarHash()) input.attrs.insert_or_assign("narHash", getNarHash()->to_string(HashFormat::SRI, true)); if (originalInput.getNarHash() && *getNarHash() != *originalInput.getNarHash()) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index e335fe6f3..261d65d69 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -163,10 +163,11 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] -if [[ $(nix config show lazy-trees) = false ]]; then - [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] -else - [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +if [[ $(nix config show lazy-trees) = true ]]; then + # Test that `lazy-locks` causes NAR hashes to be omitted from the lock file. + nix flake update --flake "$flake2Dir" --commit-lock-file --lazy-locks + [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] fi # Rerunning the build should not change the lockfile. From ff5f65dac85217ed0218f98a7cc25dd597b97795 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 13 Jun 2025 09:55:17 -0400 Subject: [PATCH 29/73] Move the actual vm tests / flake regressions into the generic build phase This lets these steps run in maximal parallelism. This also uses a success job to "combine" all the component jobs into a single signal. This also collapses the publish step into the ci job so we don't double-run --- .github/workflows/build.yml | 168 ++++++++++++++++++++- .github/workflows/ci.yml | 211 +++++++++++---------------- .github/workflows/upload-release.yml | 113 -------------- 3 files changed, 250 insertions(+), 242 deletions(-) delete mode 100644 .github/workflows/upload-release.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8baa6127f..971874733 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,13 @@ on: workflow_call: inputs: - os: + system: required: true type: string - system: + runner: + required: true + type: string + runner_small: required: true type: string if: @@ -15,13 +18,21 @@ on: required: false default: true type: boolean + run_vm_tests: + required: false + default: false + type: boolean + run_regression_tests: + required: false + default: false + type: boolean jobs: build: if: ${{ inputs.if }} strategy: fail-fast: false - runs-on: ${{ inputs.os }} + runs-on: ${{ inputs.runner }} timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -33,15 +44,164 @@ jobs: with: name: ${{ inputs.system }} path: ./tarball/*.xz + test: if: ${{ inputs.if && inputs.run_tests}} needs: build strategy: fail-fast: false - runs-on: ${{ inputs.os }} + runs-on: ${{ inputs.runner }} timeout-minutes: 60 steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/determinate-nix-action@main - uses: DeterminateSystems/flakehub-cache-action@main - run: nix flake check -L --system ${{ inputs.system }} + + vm_tests_smoke: + if: inputs.run_vm_tests && github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + nix build -L \ + .#hydraJobs.tests.functional_user \ + .#hydraJobs.tests.githubFlakes \ + .#hydraJobs.tests.nix-docker \ + .#hydraJobs.tests.tarballFlakes \ + ; + + vm_tests_all: + if: inputs.run_vm_tests && github.event_name == 'merge_group' + needs: build + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + nix build -L --keep-going \ + $(nix flake show --json \ + | jq -r ' + .hydraJobs.tests + | with_entries(select(.value.type == "derivation")) + | keys[] + | ".#hydraJobs.tests." + .') + + flake_regressions: + if: | + (inputs.run_regression_tests && github.event_name == 'merge_group') + || ( + github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) + ) + ) + needs: build + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" flake-regressions/eval-all.sh + + flake_regressions_lazy: + if: | + (inputs.run_regression_tests && github.event_name == 'merge_group') + || ( + github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) + ) + ) + needs: build + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" NIX_CONFIG="lazy-trees = true" flake-regressions/eval-all.sh + + manual: + if: github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner_small }} + permissions: + id-token: "write" + contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - name: Build manual + run: nix build .#hydraJobs.manual + - uses: nwtgck/actions-netlify@v3.0 + with: + publish-dir: "./result/share/doc/nix/manual" + production-branch: detsys-main + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy-message: "Deploy from GitHub Actions" + # NOTE(cole-h): We have a perpetual PR displaying our changes against upstream open, but + # its conversation is locked, so this PR comment can never be posted. + # https://github.com/DeterminateSystems/nix-src/pull/4 + enable-pull-request-comment: ${{ github.event.pull_request.number != 4 }} + enable-commit-comment: true + enable-commit-status: true + overwrites-pull-request-comment: true + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + + success: + needs: + - build + - test + - vm_tests_smoke + - vm_tests_all + - flake_regressions + - flake_regressions_lazy + - manual + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - run: "true" + - run: | + echo "A dependent in the build matrix failed:" + echo "$needs" + exit 1 + env: + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fcbf9360..7507d377e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,14 +4,22 @@ on: pull_request: push: branches: + # NOTE: make sure any branches here are also valid directory names, + # otherwise creating the directory and uploading to s3 will fail - detsys-main - main - master merge_group: + release: + types: + - published permissions: id-token: "write" contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" jobs: eval: @@ -26,150 +34,103 @@ jobs: build_x86_64-linux: uses: ./.github/workflows/build.yml with: - os: blacksmith-32vcpu-ubuntu-2204 system: x86_64-linux + runner: blacksmith-32vcpu-ubuntu-2204 + runner_small: ubuntu-latest + run_tests: true + run_vm_tests: true + run_regression_tests: true build_aarch64-linux: uses: ./.github/workflows/build.yml with: if: ${{ github.event_name == 'merge_group' }} - os: blacksmith-32vcpu-ubuntu-2204-arm system: aarch64-linux + runner: blacksmith-32vcpu-ubuntu-2204-arm + runner_small: blacksmith-32vcpu-ubuntu-2204-arm build_x86_64-darwin: uses: ./.github/workflows/build.yml with: if: ${{ github.event_name == 'merge_group' }} - os: macos-latest-large system: x86_64-darwin + runner: macos-latest-large + runner_small: macos-latest-large build_aarch64-darwin: uses: ./.github/workflows/build.yml with: - os: namespace-profile-mac-m2-12c28g system: aarch64-darwin + runner: namespace-profile-mac-m2-12c28g + runner_small: macos-latest-xlarge - vm_tests_smoke: - if: github.event_name != 'merge_group' - needs: build_x86_64-linux - runs-on: blacksmith-32vcpu-ubuntu-2204 + success: + runs-on: ubuntu-latest + needs: + - eval + - build_x86_64-linux + - build_aarch64-linux + - build_x86_64-darwin + - build_aarch64-darwin + if: ${{ always() }} steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main + - run: "true" - run: | - nix build -L \ - .#hydraJobs.tests.functional_user \ - .#hydraJobs.tests.githubFlakes \ - .#hydraJobs.tests.nix-docker \ - .#hydraJobs.tests.tarballFlakes \ - ; - - vm_tests_all: - if: github.event_name == 'merge_group' - needs: build_x86_64-linux - runs-on: blacksmith-32vcpu-ubuntu-2204 - steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main - - run: | - nix build -L --keep-going \ - $(nix flake show --json \ - | jq -r ' - .hydraJobs.tests - | with_entries(select(.value.type == "derivation")) - | keys[] - | ".#hydraJobs.tests." + .') - - flake_regressions: - if: | - github.event_name == 'merge_group' - || ( - github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' - && ( - (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') - || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) - ) - ) - needs: build_x86_64-linux - runs-on: namespace-profile-x86-32cpu-64gb - steps: - - name: Checkout nix - uses: actions/checkout@v4 - - name: Checkout flake-regressions - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions - path: flake-regressions - - name: Checkout flake-regressions-data - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions-data - path: flake-regressions/tests - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main - - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" flake-regressions/eval-all.sh - - flake_regressions_lazy: - if: | - github.event_name == 'merge_group' - || ( - github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' - && ( - (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') - || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) - ) - ) - needs: build_x86_64-linux - runs-on: namespace-profile-x86-32cpu-64gb - steps: - - name: Checkout nix - uses: actions/checkout@v4 - - name: Checkout flake-regressions - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions - path: flake-regressions - - name: Checkout flake-regressions-data - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions-data - path: flake-regressions/tests - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main - - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" NIX_CONFIG="lazy-trees = true" flake-regressions/eval-all.sh - - manual: - if: github.event_name != 'merge_group' - needs: build_x86_64-linux - runs-on: blacksmith - permissions: - id-token: "write" - contents: "read" - pull-requests: "write" - statuses: "write" - deployments: "write" - steps: - - name: Checkout nix - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main - - name: Build manual - run: nix build .#hydraJobs.manual - - uses: nwtgck/actions-netlify@v3.0 - with: - publish-dir: "./result/share/doc/nix/manual" - production-branch: detsys-main - github-token: ${{ secrets.GITHUB_TOKEN }} - deploy-message: "Deploy from GitHub Actions" - # NOTE(cole-h): We have a perpetual PR displaying our changes against upstream open, but - # its conversation is locked, so this PR comment can never be posted. - # https://github.com/DeterminateSystems/nix-src/pull/4 - enable-pull-request-comment: ${{ github.event.pull_request.number != 4 }} - enable-commit-comment: true - enable-commit-status: true - overwrites-pull-request-comment: true + echo "A dependent in the build matrix failed:" + echo "$needs" + exit 1 env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') + + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + + - name: Create artifacts directory + run: mkdir -p ./artifacts + + - name: Fetch artifacts + uses: actions/download-artifact@v4 + with: + path: downloaded + - name: Move downloaded artifacts to artifacts directory + run: | + for dir in ./downloaded/*; do + arch="$(basename "$dir")" + mv "$dir"/*.xz ./artifacts/"${arch}" + done + + - name: Build fallback-paths.nix + run: | + nix build .#fallbackPathsNix --out-link fallback + cat fallback > ./artifacts/fallback-paths.nix + + - uses: DeterminateSystems/push-artifact-ids@main + with: + s3_upload_role: ${{ secrets.AWS_S3_UPLOAD_ROLE_ARN }} + bucket: ${{ secrets.AWS_S3_UPLOAD_BUCKET_NAME }} + directory: ./artifacts + ids_project_name: determinate-nix + ids_binary_prefix: determinate-nix + skip_acl: true + allowed_branches: '["detsys-main"]' + + publish: + needs: + - success + if: (!github.repository.fork && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/'))) + environment: ${{ github.event_name == 'release' && 'production' || '' }} + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-push@main + with: + rolling: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + visibility: "public" + tag: "${{ github.ref_name }}" diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml deleted file mode 100644 index 9e173c34f..000000000 --- a/.github/workflows/upload-release.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Upload release - -concurrency: - group: upload-release - -on: - workflow_call: - push: - branches: - # NOTE: make sure any branches here are also valid directory names, - # otherwise creating the directory and uploading to s3 will fail - - "detsys-main" - pull_request: - types: - - opened - - reopened - - synchronize - - labeled - release: - types: - - published - -permissions: - id-token: "write" - contents: "read" - -jobs: - build-x86_64-linux: - uses: ./.github/workflows/build.yml - with: - os: blacksmith-32vcpu-ubuntu-2204 - system: x86_64-linux - run_tests: false - - build-aarch64-linux: - uses: ./.github/workflows/build.yml - with: - os: blacksmith-32vcpu-ubuntu-2204-arm - system: aarch64-linux - run_tests: false - - build-x86_64-darwin: - uses: ./.github/workflows/build.yml - with: - os: macos-latest-large - system: x86_64-darwin - run_tests: false - - build-aarch64-darwin: - uses: ./.github/workflows/build.yml - with: - os: macos-latest-xlarge - system: aarch64-darwin - run_tests: false - - release: - runs-on: ubuntu-latest - needs: - - build-x86_64-linux - - build-aarch64-linux - - build-x86_64-darwin - - build-aarch64-darwin - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - - name: Create artifacts directory - run: mkdir -p ./artifacts - - - name: Fetch artifacts - uses: actions/download-artifact@v4 - with: - path: downloaded - - name: Move downloaded artifacts to artifacts directory - run: | - for dir in ./downloaded/*; do - arch="$(basename "$dir")" - mv "$dir"/*.xz ./artifacts/"${arch}" - done - - - name: Build fallback-paths.nix - run: | - nix build .#fallbackPathsNix --out-link fallback - cat fallback > ./artifacts/fallback-paths.nix - - - uses: DeterminateSystems/push-artifact-ids@main - with: - s3_upload_role: ${{ secrets.AWS_S3_UPLOAD_ROLE_ARN }} - bucket: ${{ secrets.AWS_S3_UPLOAD_BUCKET_NAME }} - directory: ./artifacts - ids_project_name: determinate-nix - ids_binary_prefix: determinate-nix - skip_acl: true - allowed_branches: '["detsys-main"]' - - publish: - needs: - - release - if: (!github.repository.fork && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/'))) - environment: ${{ github.event_name == 'release' && 'production' || '' }} - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write - steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-push@main - with: - rolling: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} - visibility: "public" - tag: "${{ github.ref_name }}" From b2c762cd2f204624cb1e6ee9df8495942db28742 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Jun 2025 21:03:58 +0200 Subject: [PATCH 30/73] Apply suggestions from code review Co-authored-by: Luc Perkins --- src/libexpr/include/nix/expr/eval-settings.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index cb4726837..9b7573b20 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -270,9 +270,9 @@ struct EvalSettings : Config false, "lazy-locks", R"( - If enabled, Nix will only include NAR hashes in lock file entries if they're necessary to lock the input (i.e. when there is no other attribute that allows the content to be verified, like a Git revision). + If enabled, Nix only includes NAR hashes in lock file entries if they're necessary to lock the input (i.e. when there is no other attribute that allows the content to be verified, like a Git revision). This is not backward compatible with older versions of Nix. - If disabled, lock file entries will always contain a NAR hash. + If disabled, lock file entries always contain a NAR hash. )" }; }; From 8aa7d7d5cc010170771f53995e580cef19af0e0f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 13 Jun 2025 15:27:12 -0400 Subject: [PATCH 31/73] Go back to github hosted runners... blacksmith's keep dying --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7507d377e..82f54ddf5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ permissions: jobs: eval: - runs-on: blacksmith-32vcpu-ubuntu-2204 + runs-on: UbuntuLatest32Cores128G steps: - uses: actions/checkout@v4 with: @@ -35,7 +35,7 @@ jobs: uses: ./.github/workflows/build.yml with: system: x86_64-linux - runner: blacksmith-32vcpu-ubuntu-2204 + runner: UbuntuLatest32Cores128G runner_small: ubuntu-latest run_tests: true run_vm_tests: true @@ -46,8 +46,8 @@ jobs: with: if: ${{ github.event_name == 'merge_group' }} system: aarch64-linux - runner: blacksmith-32vcpu-ubuntu-2204-arm - runner_small: blacksmith-32vcpu-ubuntu-2204-arm + runner: UbuntuLatest32Cores128GArm + runner_small: UbuntuLatest32Cores128GArm build_x86_64-darwin: uses: ./.github/workflows/build.yml From cbedb8e19a0c85747526900f184d368d4b36cdaa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Jun 2025 15:21:31 +0200 Subject: [PATCH 32/73] Fix broken fetchToStore() caching on unlocked inputs --- src/libfetchers/fetchers.cc | 2 +- tests/functional/flakes/flakes.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 6d73daa1a..7ab1f567a 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -355,7 +355,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto assert(!accessor->getFingerprint(CanonPath::root)); - if (auto fingerprint = getFingerprint(store)) + if (auto fingerprint = result.getFingerprint(store)) accessor->setFingerprint(*fingerprint); return {accessor, std::move(result)}; diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index e335fe6f3..51f1909a2 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -112,6 +112,12 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" +# Check that the fetcher cache works. +if [[ $(nix config show lazy-trees) = false ]]; then + nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuietInverse "source path.*is uncacheable" + nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "store path cache hit" +fi + # Check that relative paths are allowed for git flakes. # This may change in the future once git submodule support is refined. # See: https://discourse.nixos.org/t/57783 and #9708. From 8b9cb382e97a2e60cbfe3d5df1bca0230dbc9c07 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Jun 2025 20:38:26 +0200 Subject: [PATCH 33/73] Fix `path` field in fetcher cache 86785fd9d1e8f2ed5d670e4c8bd64189af9b94a7 was broken because it was storing the full path in the MountedSourceAccessor as the `path` field in the fetcher cache key (i.e. including the /nix/store/... prefix). Especially in the case of lazy (virtual) store paths, this didn't work at all because those paths are different every time. --- src/libfetchers/fetch-to-store.cc | 10 +++-- src/libfetchers/fetchers.cc | 9 ++--- src/libfetchers/filtering-source-accessor.cc | 9 ++--- .../nix/fetchers/filtering-source-accessor.hh | 4 +- .../nix/util/forwarding-source-accessor.hh | 10 ----- .../include/nix/util/source-accessor.hh | 38 ++++++++++--------- src/libutil/mounted-source-accessor.cc | 7 ++-- 7 files changed, 38 insertions(+), 49 deletions(-) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 9a861a11d..618f32cae 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -31,10 +31,14 @@ StorePath fetchToStore( // a `PosixSourceAccessor` pointing to a store path. std::optional cacheKey; - std::optional fingerprint; - if (!filter && (fingerprint = path.accessor->getFingerprint(path.path))) { - cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, path.path.abs()); + auto [subpath, fingerprint] = + filter + ? std::pair>{path.path, std::nullopt} + : path.accessor->getFingerprint(path.path); + + if (fingerprint) { + cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs()); if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store, mode == FetchMode::DryRun)) { debug("store path cache hit for '%s'", path); return res->storePath; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7ab1f567a..5764f310d 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -338,8 +338,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto accessor = make_ref(makeStorePathAccessor(store, storePath)); - if (auto fingerprint = getFingerprint(store)) - accessor->setFingerprint(*fingerprint); + accessor->fingerprint = getFingerprint(store); // FIXME: ideally we would use the `showPath()` of the // "real" accessor for this fetcher type. @@ -353,10 +352,8 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, result] = scheme->getAccessor(store, *this); - assert(!accessor->getFingerprint(CanonPath::root)); - - if (auto fingerprint = result.getFingerprint(store)) - accessor->setFingerprint(*fingerprint); + assert(!accessor->fingerprint); + accessor->fingerprint = result.getFingerprint(store); return {accessor, std::move(result)}; } diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 12e4a688b..c339cdbdb 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -58,16 +58,13 @@ std::string FilteringSourceAccessor::showPath(const CanonPath & path) return displayPrefix + next->showPath(prefix / path) + displaySuffix; } -std::optional FilteringSourceAccessor::getFingerprint(const CanonPath & path) +std::pair> FilteringSourceAccessor::getFingerprint(const CanonPath & path) { + if (fingerprint) + return {path, fingerprint}; return next->getFingerprint(prefix / path); } -void FilteringSourceAccessor::setFingerprint(std::string fingerprint) -{ - next->setFingerprint(std::move(fingerprint)); -} - void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 391cd371b..e0228ad9b 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -50,9 +50,7 @@ struct FilteringSourceAccessor : SourceAccessor std::string showPath(const CanonPath & path) override; - std::optional getFingerprint(const CanonPath & path) override; - - void setFingerprint(std::string fingerprint) override; + std::pair> getFingerprint(const CanonPath & path) override; /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` diff --git a/src/libutil/include/nix/util/forwarding-source-accessor.hh b/src/libutil/include/nix/util/forwarding-source-accessor.hh index cfa5ff9b8..bdba2addc 100644 --- a/src/libutil/include/nix/util/forwarding-source-accessor.hh +++ b/src/libutil/include/nix/util/forwarding-source-accessor.hh @@ -52,16 +52,6 @@ struct ForwardingSourceAccessor : SourceAccessor { return next->getPhysicalPath(path); } - - std::optional getFingerprint(const CanonPath & path) override - { - return next->getFingerprint(path); - } - - void setFingerprint(std::string fingerprint) override - { - next->setFingerprint(std::move(fingerprint)); - } }; } diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 560e1fda0..4084b3bdc 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -177,28 +177,32 @@ struct SourceAccessor : std::enable_shared_from_this SymlinkResolution mode = SymlinkResolution::Full); /** - * Return a string that uniquely represents the contents of this - * accessor. This is used for caching lookups (see - * `fetchToStore()`). - * - * Fingerprints are generally for the entire accessor, but this - * method takes a `path` argument to support accessors like - * `MountedSourceAccessor` that combine multiple underlying - * accessors. A fingerprint should only be returned if it uniquely - * represents everything under `path`. + * A string that uniquely represents the contents of this + * accessor. This is used for caching lookups (see `fetchToStore()`). */ - virtual std::optional getFingerprint(const CanonPath & path) - { - return _fingerprint; - } + std::optional fingerprint; - virtual void setFingerprint(std::string fingerprint) + /** + * Return the fingerprint for `path`. This is usually the + * fingerprint of the current accessor, but for composite + * accessors (like `MountedSourceAccessor`), we want to return the + * fingerprint of the "inner" accessor if the current one lacks a + * fingerprint. + * + * So this method is intended to return the most-outer accessor + * that has a fingerprint for `path`. It also returns the path that `path` + * corresponds to in that accessor. + * + * For example: in a `MountedSourceAccessor` that has + * `/nix/store/foo` mounted, + * `getFingerprint("/nix/store/foo/bar")` will return the path + * `/bar` and the fingerprint of the `/nix/store/foo` accessor. + */ + virtual std::pair> getFingerprint(const CanonPath & path) { - _fingerprint = std::move(fingerprint); + return {path, fingerprint}; } - std::optional _fingerprint; - /** * Return the maximum last-modified time of the files in this * tree, if available. diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index 9292291c1..ed62fd2a3 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -91,12 +91,11 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor return nullptr; } - std::optional getFingerprint(const CanonPath & path) override + std::pair> getFingerprint(const CanonPath & path) override { + if (fingerprint) + return {path, fingerprint}; auto [accessor, subpath] = resolve(path); - // FIXME: check that there are no mounts underneath the mount - // point of `accessor`, since that would invalidate the - // fingerprint. (However we don't have such at the moment.) return accessor->getFingerprint(subpath); } }; From c7d80871accebd87495aa8432b21bdd88fc0c827 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 13 Jun 2025 13:46:31 -0400 Subject: [PATCH 34/73] Parallelize the flake regression suite --- .github/workflows/build.yml | 52 ++++++++++++++----------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 971874733..8ab836a36 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -95,7 +95,8 @@ jobs: if: | (inputs.run_regression_tests && github.event_name == 'merge_group') || ( - github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + inputs.run_regression_tests + && github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' && ( (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) @@ -103,6 +104,16 @@ jobs: ) needs: build runs-on: ${{ inputs.runner }} + strategy: + matrix: + nix_config: + - "lazy-trees = true" + - "lazy-trees = false" + glob: + - "[0-d]*" + - "[e-l]*" + - "[m-r]*" + - "[s-z]*" steps: - name: Checkout nix uses: actions/checkout@v4 @@ -118,36 +129,14 @@ jobs: path: flake-regressions/tests - uses: DeterminateSystems/determinate-nix-action@main - uses: DeterminateSystems/flakehub-cache-action@main - - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" flake-regressions/eval-all.sh - - flake_regressions_lazy: - if: | - (inputs.run_regression_tests && github.event_name == 'merge_group') - || ( - github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' - && ( - (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') - || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) - ) - ) - needs: build - runs-on: ${{ inputs.runner }} - steps: - - name: Checkout nix - uses: actions/checkout@v4 - - name: Checkout flake-regressions - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions - path: flake-regressions - - name: Checkout flake-regressions-data - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions-data - path: flake-regressions/tests - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main - - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" NIX_CONFIG="lazy-trees = true" flake-regressions/eval-all.sh + - env: + PARALLEL: "-P 50%" + FLAKE_REGRESSION_GLOB: ${{ matrix.glob }} + NIX_CONFIG: ${{ matrix.nix_config }} + run: | + nix build -L --out-link ./new-nix + export PATH=$(pwd)/new-nix/bin:$PATH + flake-regressions/eval-all.sh manual: if: github.event_name != 'merge_group' @@ -190,7 +179,6 @@ jobs: - vm_tests_smoke - vm_tests_all - flake_regressions - - flake_regressions_lazy - manual if: ${{ always() }} runs-on: ubuntu-latest From f764c9eae37c1d31133b7ff209e544d3381ec803 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 13 Jun 2025 14:58:00 -0400 Subject: [PATCH 35/73] Add retry to the eval steps, split out m since it takes a while on its own --- .github/workflows/build.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ab836a36..c3ae8fce1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -112,7 +112,8 @@ jobs: glob: - "[0-d]*" - "[e-l]*" - - "[m-r]*" + - "[m]*" + - "[n-r]*" - "[s-z]*" steps: - name: Checkout nix @@ -136,7 +137,12 @@ jobs: run: | nix build -L --out-link ./new-nix export PATH=$(pwd)/new-nix/bin:$PATH - flake-regressions/eval-all.sh + + if ! flake-regressions/eval-all.sh; then + echo "Some failed, trying again" + printf "\n\n\n\n\n\n\n\n" + flake-regressions/eval-all.sh + fi manual: if: github.event_name != 'merge_group' From b0a325d8db1adc89f2b66ee75e0bd162ce0643da Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 13 Jun 2025 15:18:04 -0400 Subject: [PATCH 36/73] Run on GHA runners, not blacksmith due to timeouts From 6185afef92cda34416d4ad97251b2e4ad8141cbb Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 13 Jun 2025 15:24:01 -0400 Subject: [PATCH 37/73] Run onnamespace, use their cache --- .github/workflows/build.yml | 5 +++++ .github/workflows/ci.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c3ae8fce1..24f0c30c1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -135,6 +135,11 @@ jobs: FLAKE_REGRESSION_GLOB: ${{ matrix.glob }} NIX_CONFIG: ${{ matrix.nix_config }} run: | + set -x + if [ ! -z "${NSC_CACHE_PATH:-}" ]; then + mkdir -p "${NSC_CACHE_PATH}/nix/xdg-cache" + export XDG_CACHE_HOME="${NSC_CACHE_PATH}/nix/xdg-cache" + fi nix build -L --out-link ./new-nix export PATH=$(pwd)/new-nix/bin:$PATH diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82f54ddf5..832228296 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: uses: ./.github/workflows/build.yml with: system: x86_64-linux - runner: UbuntuLatest32Cores128G + runner: namespace-profile-linuxamd32c64g-cache runner_small: ubuntu-latest run_tests: true run_vm_tests: true From 97dc226cae99c5c4573dd706c4dc5137e64adcc8 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 13 Jun 2025 16:43:55 -0400 Subject: [PATCH 38/73] Use fewer runners --- .github/workflows/build.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 24f0c30c1..36dcab930 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,11 +110,8 @@ jobs: - "lazy-trees = true" - "lazy-trees = false" glob: - - "[0-d]*" - - "[e-l]*" - - "[m]*" - - "[n-r]*" - - "[s-z]*" + - "[0-l]*" + - "[m-z]*" steps: - name: Checkout nix uses: actions/checkout@v4 From cf11e27047a418a42004493a8f18ab50ef1e424a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 13 Jun 2025 18:05:15 -0400 Subject: [PATCH 39/73] Don't stall on a stuck nixos vm test build forever --- .github/workflows/build.yml | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36dcab930..ef6d9072e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,13 +83,21 @@ jobs: - uses: DeterminateSystems/determinate-nix-action@main - uses: DeterminateSystems/flakehub-cache-action@main - run: | - nix build -L --keep-going \ - $(nix flake show --json \ - | jq -r ' - .hydraJobs.tests - | with_entries(select(.value.type == "derivation")) - | keys[] - | ".#hydraJobs.tests." + .') + cmd() { + nix build -L --keep-going --timeout 300 \ + $(nix flake show --json \ + | jq -r ' + .hydraJobs.tests + | with_entries(select(.value.type == "derivation")) + | keys[] + | ".#hydraJobs.tests." + .') + } + + if ! cmd; then + echo "failed, retrying once ..." + printf "\n\n\n\n\n\n\n\n" + cmd + fi flake_regressions: if: | @@ -110,8 +118,12 @@ jobs: - "lazy-trees = true" - "lazy-trees = false" glob: - - "[0-l]*" - - "[m-z]*" + - "[0-d]*" + - "[e-l]*" + - "[m]*" + - "[n-r]*" + - "[s-z]*" + steps: - name: Checkout nix uses: actions/checkout@v4 From 151456a999de3c32d86b4091fc0cff4b8a2255a5 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Jun 2025 09:54:06 -0400 Subject: [PATCH 40/73] Change the aarch64 linux / x86 darwin condition to exclude pull request, since we do want to build them on every other trigger --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 832228296..4aa9ed296 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: build_aarch64-linux: uses: ./.github/workflows/build.yml with: - if: ${{ github.event_name == 'merge_group' }} + if: ${{ github.event_name != 'pull_request' }} system: aarch64-linux runner: UbuntuLatest32Cores128GArm runner_small: UbuntuLatest32Cores128GArm @@ -52,7 +52,7 @@ jobs: build_x86_64-darwin: uses: ./.github/workflows/build.yml with: - if: ${{ github.event_name == 'merge_group' }} + if: ${{ github.event_name != 'pull_request' }} system: x86_64-darwin runner: macos-latest-large runner_small: macos-latest-large From eba6cd8488f5a502fd52ea5ffa38bb22c6ec91e9 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Jun 2025 09:55:46 -0400 Subject: [PATCH 41/73] Don't build fallback-paths if we didn't build aarch64-linux and x86 darwin --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4aa9ed296..4eea8759d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,7 @@ jobs: done - name: Build fallback-paths.nix + if: ${{ github.event_name != 'pull_request' }} run: | nix build .#fallbackPathsNix --out-link fallback cat fallback > ./artifacts/fallback-paths.nix From dc5e6200325ad5c8f380de8777e4d4f24e0032ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Jun 2025 22:19:53 +0200 Subject: [PATCH 42/73] fetchToStore() cache: Use content hashes instead of store paths We can always compute the store path from the content hash, but not vice versa. Storing the content hash allows `hashPath()` to be replaced by `fetchToStore(...FetchMode::DryRun...)`, which gets us caching in lazy-trees mode. --- src/libexpr/paths.cc | 3 +- src/libfetchers/fetch-to-store.cc | 73 ++++++++++++------- src/libfetchers/fetchers.cc | 4 +- .../include/nix/fetchers/fetch-to-store.hh | 13 +++- src/libfetchers/path.cc | 35 +++------ tests/functional/flakes/flakes.sh | 2 +- 6 files changed, 72 insertions(+), 58 deletions(-) diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 65b8212e1..b6a372fb2 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -84,8 +84,7 @@ StorePath EvalState::mountInput( if (store->isValidPath(storePath)) _narHash = store->queryPathInfo(storePath)->narHash; else - // FIXME: use fetchToStore to make it cache this - _narHash = accessor->hashPath(CanonPath::root); + _narHash = fetchToStore2(*store, accessor, FetchMode::DryRun, input.getName()).second; } return _narHash; }; diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 618f32cae..5595f7594 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -3,19 +3,16 @@ namespace nix { -fetchers::Cache::Key makeFetchToStoreCacheKey( - const std::string &name, - const std::string &fingerprint, +fetchers::Cache::Key makeSourcePathToHashCacheKey( + const std::string & fingerprint, ContentAddressMethod method, - const std::string &path) + const std::string & path) { - return fetchers::Cache::Key{"fetchToStore", { - {"name", name}, + return fetchers::Cache::Key{"sourcePathToHash", { {"fingerprint", fingerprint}, {"method", std::string{method.render()}}, {"path", path} }}; - } StorePath fetchToStore( @@ -27,9 +24,18 @@ StorePath fetchToStore( PathFilter * filter, RepairFlag repair) { - // FIXME: add an optimisation for the case where the accessor is - // a `PosixSourceAccessor` pointing to a store path. + return fetchToStore2(store, path, mode, name, method, filter, repair).first; +} +std::pair fetchToStore2( + Store & store, + const SourcePath & path, + FetchMode mode, + std::string_view name, + ContentAddressMethod method, + PathFilter * filter, + RepairFlag repair) +{ std::optional cacheKey; auto [subpath, fingerprint] = @@ -38,32 +44,47 @@ StorePath fetchToStore( : path.accessor->getFingerprint(path.path); if (fingerprint) { - cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs()); - if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store, mode == FetchMode::DryRun)) { - debug("store path cache hit for '%s'", path); - return res->storePath; + cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath.abs()); + if (auto res = fetchers::getCache()->lookup(*cacheKey)) { + debug("source path hash cache hit for '%s'", path); + auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash")); + auto storePath = store.makeFixedOutputPathFromCA(name, + ContentAddressWithReferences::fromParts(method, hash, {})); + if (store.isValidPath(storePath)) { + debug("source path '%s' has valid store path '%s'", path, store.printStorePath(storePath)); + return {storePath, hash}; + } + debug("source path '%s' not in store", path); } } else - debug("source path '%s' is uncacheable (%d, %d)", path, filter, (bool) fingerprint); + // FIXME: could still provide in-memory caching keyed on `SourcePath`. + debug("source path '%s' is uncacheable (%d, %d)", path, (bool) filter, (bool) fingerprint); Activity act(*logger, lvlChatty, actUnknown, fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path)); auto filter2 = filter ? *filter : defaultPathFilter; - auto storePath = - mode == FetchMode::DryRun - ? store.computeStorePath( - name, path, method, HashAlgorithm::SHA256, {}, filter2).first - : store.addToStore( + if (mode == FetchMode::DryRun) { + auto [storePath, hash] = store.computeStorePath( + name, path, method, HashAlgorithm::SHA256, {}, filter2); + debug("hashed '%s' to '%s'", path, store.printStorePath(storePath)); + if (cacheKey) + fetchers::getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); + return {storePath, hash}; + } else { + auto storePath = store.addToStore( name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); - - debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath)); - - if (cacheKey) - fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); - - return storePath; + debug("copied '%s' to '%s'", path, store.printStorePath(storePath)); + // FIXME: this is the wrong hash when method != + // ContentAddressMethod::Raw::NixArchive. Doesn't matter at + // the moment since the only place where that's the case + // doesn't use the hash. + auto hash = store.queryPathInfo(storePath)->narHash; + if (cacheKey) + fetchers::getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); + return {storePath, hash}; + } } } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5764f310d..d91f24b6a 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -352,8 +352,8 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, result] = scheme->getAccessor(store, *this); - assert(!accessor->fingerprint); - accessor->fingerprint = result.getFingerprint(store); + if (!accessor->fingerprint) + accessor->fingerprint = result.getFingerprint(store); return {accessor, std::move(result)}; } diff --git a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh index 44c33c147..364d25375 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh @@ -23,7 +23,16 @@ StorePath fetchToStore( PathFilter * filter = nullptr, RepairFlag repair = NoRepair); -fetchers::Cache::Key makeFetchToStoreCacheKey( - const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path); +std::pair fetchToStore2( + Store & store, + const SourcePath & path, + FetchMode mode, + std::string_view name = "source", + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + +fetchers::Cache::Key +makeSourcePathToHashCacheKey(const std::string & fingerprint, ContentAddressMethod method, const std::string & path); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index ff39cb02f..0de81ae43 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -144,37 +144,22 @@ struct PathInputScheme : InputScheme storePath = store->addToStoreFromDump(*src, "source"); } - // To avoid copying the path again to the /nix/store, we need to add a cache entry. - ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive; - auto fp = getFingerprint(store, input); - if (fp) { - auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/"); - fetchers::getCache()->upsert(cacheKey, *store, {}, *storePath); - } + auto accessor = makeStorePathAccessor(store, *storePath); + + // To prevent `fetchToStore()` copying the path again to Nix + // store, pre-create an entry in the fetcher cache. + auto info = store->queryPathInfo(*storePath); + accessor->fingerprint = fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)); + fetchers::getCache()->upsert( + makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), + {{"hash", info->narHash.to_string(HashFormat::SRI, true)}}); /* Trust the lastModified value supplied by the user, if any. It's not a "secure" attribute so we don't care. */ if (!input.getLastModified()) input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {makeStorePathAccessor(store, *storePath), std::move(input)}; - } - - std::optional getFingerprint(ref store, const Input & input) const override - { - if (isRelative(input)) - return std::nullopt; - - /* If this path is in the Nix store, use the hash of the - store object and the subpath. */ - auto path = getAbsPath(input); - try { - auto [storePath, subPath] = store->toStorePath(path.string()); - auto info = store->queryPathInfo(storePath); - return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath); - } catch (Error &) { - return std::nullopt; - } + return {accessor, std::move(input)}; } }; diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 51f1909a2..878e02682 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -115,7 +115,7 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" # Check that the fetcher cache works. if [[ $(nix config show lazy-trees) = false ]]; then nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuietInverse "source path.*is uncacheable" - nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "store path cache hit" + nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "source path hash cache hit" fi # Check that relative paths are allowed for git flakes. From af5815fd540d4bde68f93526e1bf23e0f8b2cff1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 12:09:43 +0200 Subject: [PATCH 43/73] Give unit tests access to a $HOME directory Also, don't try to access cache.nixos.org in the libstore unit tests. --- src/libflake-tests/meson.build | 1 + src/libflake-tests/package.nix | 18 +++++++----------- src/libstore-tests/meson.build | 1 + src/libstore-tests/nix_api_store.cc | 16 +--------------- src/libstore-tests/package.nix | 18 +++++++----------- 5 files changed, 17 insertions(+), 37 deletions(-) diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 80c94bd77..b7a48b89e 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -59,6 +59,7 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + 'HOME': meson.current_build_dir() / 'test-home', }, protocol : 'gtest', ) diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index db507fc3a..8344d98d7 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -56,17 +56,13 @@ mkMesonExecutable (finalAttrs: { { meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; } - ( - lib.optionalString stdenv.hostPlatform.isWindows '' - export HOME="$PWD/home-dir" - mkdir -p "$HOME" - '' - + '' - export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} - touch $out - '' - ); + ('' + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + export HOME="$TMPDIR/home" + mkdir -p "$HOME" + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); }; }; diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 8a1ff40f0..8b9893b23 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -100,6 +100,7 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + 'HOME': meson.current_build_dir() / 'test-home', }, protocol : 'gtest', ) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 4eb95360a..b7495e0ab 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -28,10 +28,6 @@ TEST_F(nix_api_store_test, nix_store_get_uri) TEST_F(nix_api_util_context, nix_store_get_storedir_default) { - if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { - // skipping test in sandbox because nix_store_open tries to create /nix/var/nix/profiles - GTEST_SKIP(); - } nix_libstore_init(ctx); Store * store = nix_store_open(ctx, nullptr, nullptr); assert_ctx_ok(); @@ -136,10 +132,6 @@ TEST_F(nix_api_store_test, nix_store_real_path) TEST_F(nix_api_util_context, nix_store_real_path_relocated) { - if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { - // Can't open default store from within sandbox - GTEST_SKIP(); - } auto tmp = nix::createTempDir(); std::string storeRoot = tmp + "/store"; std::string stateDir = tmp + "/state"; @@ -179,13 +171,7 @@ TEST_F(nix_api_util_context, nix_store_real_path_relocated) TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) { - if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { - // TODO: override NIX_CACHE_HOME? - // skipping test in sandbox because narinfo cache can't be written - GTEST_SKIP(); - } - - Store * store = nix_store_open(ctx, "https://cache.nixos.org", nullptr); + Store * store = nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir()).c_str(), nullptr); assert_ctx_ok(); ASSERT_NE(store, nullptr); diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index b39ee7fa7..1f3701c7f 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -73,17 +73,13 @@ mkMesonExecutable (finalAttrs: { { meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; } - ( - lib.optionalString stdenv.hostPlatform.isWindows '' - export HOME="$PWD/home-dir" - mkdir -p "$HOME" - '' - + '' - export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} - ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} - touch $out - '' - ); + ('' + export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} + export HOME="$TMPDIR/home" + mkdir -p "$HOME" + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); }; }; From e3fa4faff92e6769f77fd067177336e8f74629a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 16:04:26 +0200 Subject: [PATCH 44/73] fetchToStore(): Don't require a valid path in dry run mode --- src/libfetchers/fetch-to-store.cc | 5 ++--- tests/functional/flakes/flakes.sh | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 5595f7594..31de2b1e1 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -46,12 +46,11 @@ std::pair fetchToStore2( if (fingerprint) { cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath.abs()); if (auto res = fetchers::getCache()->lookup(*cacheKey)) { - debug("source path hash cache hit for '%s'", path); auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash")); auto storePath = store.makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(method, hash, {})); - if (store.isValidPath(storePath)) { - debug("source path '%s' has valid store path '%s'", path, store.printStorePath(storePath)); + if (mode == FetchMode::DryRun || store.isValidPath(storePath)) { + debug("source path '%s' cache hit in '%s' (hash '%s')", path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); return {storePath, hash}; } debug("source path '%s' not in store", path); diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 878e02682..ddfd7052f 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -115,7 +115,7 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" # Check that the fetcher cache works. if [[ $(nix config show lazy-trees) = false ]]; then nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuietInverse "source path.*is uncacheable" - nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "source path hash cache hit" + nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "source path.*cache hit" fi # Check that relative paths are allowed for git flakes. From b2905dc08e87bfb9b3d5f238ba731d958d9b0cbd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 16:05:08 +0200 Subject: [PATCH 45/73] fetchToStore(): Address a FIXME --- src/libfetchers/fetch-to-store.cc | 51 +++++++++++++++++++------------ 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 31de2b1e1..877e49c14 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -64,26 +64,37 @@ std::pair fetchToStore2( auto filter2 = filter ? *filter : defaultPathFilter; - if (mode == FetchMode::DryRun) { - auto [storePath, hash] = store.computeStorePath( - name, path, method, HashAlgorithm::SHA256, {}, filter2); - debug("hashed '%s' to '%s'", path, store.printStorePath(storePath)); - if (cacheKey) - fetchers::getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); - return {storePath, hash}; - } else { - auto storePath = store.addToStore( - name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); - debug("copied '%s' to '%s'", path, store.printStorePath(storePath)); - // FIXME: this is the wrong hash when method != - // ContentAddressMethod::Raw::NixArchive. Doesn't matter at - // the moment since the only place where that's the case - // doesn't use the hash. - auto hash = store.queryPathInfo(storePath)->narHash; - if (cacheKey) - fetchers::getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); - return {storePath, hash}; - } + auto [storePath, hash] = + mode == FetchMode::DryRun + ? ({ + auto [storePath, hash] = store.computeStorePath( + name, path, method, HashAlgorithm::SHA256, {}, filter2); + debug("hashed '%s' to '%s' (hash '%s')", path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); + std::make_pair(storePath, hash); + }) + : ({ + // FIXME: ideally addToStore() would return the hash + // right away (like computeStorePath()). + auto storePath = store.addToStore( + name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); + auto info = store.queryPathInfo(storePath); + assert(info->references.empty()); + auto hash = + method == ContentAddressMethod::Raw::NixArchive + ? info->narHash + : ({ + if (!info->ca || info->ca->method != method) + throw Error("path '%s' lacks a CA field", store.printStorePath(storePath)); + info->ca->hash; + }); + debug("copied '%s' to '%s' (hash '%s')", path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); + std::make_pair(storePath, hash); + }); + + if (cacheKey) + fetchers::getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); + + return {storePath, hash}; } } From 1aadf1e96ccf1d7aa966cdac66dfb39bd3f22b10 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Jun 2025 13:29:47 -0400 Subject: [PATCH 46/73] Increase the nixos test timeout to 10 minutes, up from 5 Most tests complete within 4m, one test -- the docker test -- takes approximately 6m45s. Ten gives us plenty of room ...? --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef6d9072e..a0d6d9f98 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,7 +84,7 @@ jobs: - uses: DeterminateSystems/flakehub-cache-action@main - run: | cmd() { - nix build -L --keep-going --timeout 300 \ + nix build -L --keep-going --timeout 600 \ $(nix flake show --json \ | jq -r ' .hydraJobs.tests From 20742e63e2dbe47f330e4d67ff49e93e6b9ae458 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Jun 2025 16:07:28 -0400 Subject: [PATCH 47/73] Use GHA runners for VMs since they have KVM --- .github/workflows/build.yml | 7 +++++-- .github/workflows/ci.yml | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef6d9072e..b8844c60d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,9 @@ on: runner: required: true type: string + runner_for_virt: + required: true + type: string runner_small: required: true type: string @@ -61,7 +64,7 @@ jobs: vm_tests_smoke: if: inputs.run_vm_tests && github.event_name != 'merge_group' needs: build - runs-on: ${{ inputs.runner }} + runs-on: ${{ inputs.runner_for_virt }} steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/determinate-nix-action@main @@ -77,7 +80,7 @@ jobs: vm_tests_all: if: inputs.run_vm_tests && github.event_name == 'merge_group' needs: build - runs-on: ${{ inputs.runner }} + runs-on: ${{ inputs.runner_for_virt }} steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/determinate-nix-action@main diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4eea8759d..c002d0b66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: with: system: x86_64-linux runner: namespace-profile-linuxamd32c64g-cache + runner_for_virt: UbuntuLatest32Cores128G runner_small: ubuntu-latest run_tests: true run_vm_tests: true @@ -47,6 +48,7 @@ jobs: if: ${{ github.event_name != 'pull_request' }} system: aarch64-linux runner: UbuntuLatest32Cores128GArm + runner_for_virt: UbuntuLatest32Cores128GArm runner_small: UbuntuLatest32Cores128GArm build_x86_64-darwin: @@ -55,6 +57,7 @@ jobs: if: ${{ github.event_name != 'pull_request' }} system: x86_64-darwin runner: macos-latest-large + runner_for_virt: macos-latest-large runner_small: macos-latest-large build_aarch64-darwin: @@ -62,6 +65,7 @@ jobs: with: system: aarch64-darwin runner: namespace-profile-mac-m2-12c28g + runner_for_virt: namespace-profile-mac-m2-12c28g runner_small: macos-latest-xlarge success: From 2190f6ce6686817acdea467f3e3324b62de1152d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 21:08:34 +0000 Subject: [PATCH 48/73] Prepare release v3.6.5 From 11d72504486c7ec8b1cd749e7c077a418a4d95ec Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 21:08:37 +0000 Subject: [PATCH 49/73] Set .version-determinate to 3.6.5 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 0f44168a4..d15b8b06f 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.4 +3.6.5 From fd4ed4c9f56d9444caf73280979a9c11111397e5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 21:08:42 +0000 Subject: [PATCH 50/73] Generare release notes for 3.6.5 --- doc/manual/source/SUMMARY.md.in | 1 + .../release-notes-determinate/changes.md | 20 ++++++++++++++++++- .../release-notes-determinate/rl-3.6.5.md | 16 +++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.5.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 781dba88c..18d8c13b0 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) - [Release 3.6.4 (2025-06-12)](release-notes-determinate/rl-3.6.4.md) - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 37ff16592..ca75f9fd5 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.4. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.5. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -71,3 +71,21 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + + + +* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) + +* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) + +* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) + +* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) + +* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) + +* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) + +* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) + +* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) \ No newline at end of file diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.5.md b/doc/manual/source/release-notes-determinate/rl-3.6.5.md new file mode 100644 index 000000000..d7e7b8c07 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.5.md @@ -0,0 +1,16 @@ +# Release 3.6.5 (2025-06-16) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) +* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) +* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) +* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) +* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) +* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) +* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) +* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) + + +**Full Changelog**: [v3.6.3...v3.6.5](https://github.com/DeterminateSystems/nix-src/compare/v3.6.3...v3.6.5) From fe6972f6f0b030ffccc20f6fb203911550dd2254 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Jun 2025 17:16:14 -0400 Subject: [PATCH 51/73] Fixup type (generare) --- .github/workflows/propose-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/propose-release.yml b/.github/workflows/propose-release.yml index 82407abe7..ea01e4b7a 100644 --- a/.github/workflows/propose-release.yml +++ b/.github/workflows/propose-release.yml @@ -29,4 +29,4 @@ jobs: git commit -m "Set .version-determinate to ${{ inputs.version }}" || true ./.github/release-notes.sh git add doc - git commit -m "Generare release notes for ${{ inputs.version }}" || true + git commit -m "Generate release notes for ${{ inputs.version }}" || true From 97e2e9159b6217a33b96df58f8edce8f6e03a408 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Jun 2025 17:17:42 -0400 Subject: [PATCH 52/73] Fixup generar->te in commit automation From c0e53f3312f9a743f6997a83e9dddf21aaaf853f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Jun 2025 17:17:55 -0400 Subject: [PATCH 53/73] Move the 3.6.4 notes over --- doc/manual/source/SUMMARY.md.in | 1 - .../release-notes-determinate/changes.md | 21 ++-------------- .../release-notes-determinate/rl-3.6.4.md | 20 --------------- .../release-notes-determinate/rl-3.6.5.md | 25 +++++++++++-------- 4 files changed, 16 insertions(+), 51 deletions(-) delete mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.4.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 18d8c13b0..374aacb59 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -130,7 +130,6 @@ - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) - - [Release 3.6.4 (2025-06-12)](release-notes-determinate/rl-3.6.4.md) - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) - [Release 3.6.0 (2025-05-22)](release-notes-determinate/rl-3.6.0.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index ca75f9fd5..30a68f6e9 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -47,8 +47,9 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Call out that `--keep-failed` with remote builders will keep the failed build directory on that builder by @cole-h in [DeterminateSystems/nix-src#85](https://github.com/DeterminateSystems/nix-src/pull/85) - + + * When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) @@ -71,21 +72,3 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) - - - -* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) - -* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) - -* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) - -* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) - -* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) - -* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) - -* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) - -* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) \ No newline at end of file diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.4.md b/doc/manual/source/release-notes-determinate/rl-3.6.4.md deleted file mode 100644 index 64086bfba..000000000 --- a/doc/manual/source/release-notes-determinate/rl-3.6.4.md +++ /dev/null @@ -1,20 +0,0 @@ -# Release 3.6.4 (2025-06-12) - -* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). - -## What's Changed -* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) -* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) -* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) -* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) -* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) -* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) -* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) -* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) -* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) -* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) -* Run the Nix test suite against lazy trees by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) -* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) - - -**Full Changelog**: [v3.6.2...v3.6.4](https://github.com/DeterminateSystems/nix-src/compare/v3.6.2...v3.6.4) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.5.md b/doc/manual/source/release-notes-determinate/rl-3.6.5.md index d7e7b8c07..8ef5be0fd 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.5.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.5.md @@ -1,16 +1,19 @@ -# Release 3.6.5 (2025-06-16) +# Release 3.6.5 (2025-06-12) * Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). ## What's Changed -* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) -* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) -* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) -* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) -* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) -* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) -* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) -* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) +* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) +* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) +* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) +* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) +* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) +* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) +* Run the Nix test suite against lazy trees by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) +* Improve caching of inputs by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98), [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110), and [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) - -**Full Changelog**: [v3.6.3...v3.6.5](https://github.com/DeterminateSystems/nix-src/compare/v3.6.3...v3.6.5) +**Full Changelog**: [v3.6.2...v3.6.5](https://github.com/DeterminateSystems/nix-src/compare/v3.6.2...v3.6.4) From 0f3892185d00165cb0d326e8ba0bc13da4b65db4 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Jun 2025 17:20:58 -0400 Subject: [PATCH 54/73] Add a final newline in the generated release notes --- .github/release-notes.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/release-notes.sh b/.github/release-notes.sh index 9937c18cf..198361161 100755 --- a/.github/release-notes.sh +++ b/.github/release-notes.sh @@ -45,6 +45,7 @@ linkify_gh() { | trim_trailing_newlines \ | sed -e 's/^\* /\n* /' \ | linkify_gh + echo "" # final newline ) > "$scratch/changes.md" ( From 5329a45ade7fd94c180a538e0e7832fbff1220e7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 18:16:30 +0200 Subject: [PATCH 55/73] Git fetcher: Make dirty repos with no commits cacheable --- src/libfetchers/git.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 2825b72ab..4a00d4e34 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -860,7 +860,7 @@ struct GitInputScheme : InputScheme return makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); - if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { + if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) { /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; @@ -873,7 +873,7 @@ struct GitInputScheme : InputScheme writeString("deleted:", hashSink); writeString(file.abs(), hashSink); } - return makeFingerprint(*repoInfo.workdirInfo.headRev) + return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev)) + ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false); } return std::nullopt; From a175c67def61f68b8b1ac860ea7b0672badd1d6e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 19:34:28 +0200 Subject: [PATCH 56/73] Fix rootFS fingerprint in the impure case --- src/libutil/union-source-accessor.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index 9950f6049..69cf04c18 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -72,6 +72,18 @@ struct UnionSourceAccessor : SourceAccessor } return std::nullopt; } + + std::pair> getFingerprint(const CanonPath & path) override + { + if (fingerprint) + return {path, fingerprint}; + for (auto & accessor : accessors) { + auto [subpath, fingerprint] = accessor->getFingerprint(path); + if (fingerprint) + return {subpath, fingerprint}; + } + return {path, std::nullopt}; + } }; ref makeUnionSourceAccessor(std::vector> && accessors) From 3a4e6cadebacba2723306f2da7f9bdd12ccdf5ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 18:25:55 +0200 Subject: [PATCH 57/73] Improve regression testing for uncachable source paths These now throw an error by default in the test suite. --- src/libfetchers/fetch-to-store.cc | 8 ++++++-- tests/functional/flakes/common.sh | 2 ++ tests/functional/flakes/flake-in-submodule.sh | 6 +++--- tests/functional/flakes/flakes.sh | 1 - tests/functional/flakes/mercurial.sh | 4 ++-- tests/functional/flakes/non-flake-inputs.sh | 8 ++++---- tests/functional/flakes/relative-paths-lockfile.sh | 2 ++ 7 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 877e49c14..e6b9430a2 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -55,9 +55,13 @@ std::pair fetchToStore2( } debug("source path '%s' not in store", path); } - } else + } else { + static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1"; + if (barf) + throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter); // FIXME: could still provide in-memory caching keyed on `SourcePath`. - debug("source path '%s' is uncacheable (%d, %d)", path, (bool) filter, (bool) fingerprint); + debug("source path '%s' is uncacheable", path); + } Activity act(*logger, lvlChatty, actUnknown, fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path)); diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 422cab96c..77bc03060 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -2,6 +2,8 @@ source ../common.sh +export _NIX_TEST_BARF_ON_UNCACHEABLE=1 + # shellcheck disable=SC2034 # this variable is used by tests that source this file registry=$TEST_ROOT/registry.json diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index fe5acf26d..a7d86698d 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -62,8 +62,8 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule # Check that dirtying a submodule makes the entire thing dirty. [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]] echo '"foo"' > "$rootRepo"/submodule/sub.nix -[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] -[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval --json "$flakeref#sub" ) = '"foo"' ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] # Test that `nix flake metadata` parses `submodule` correctly. cat > "$rootRepo"/flake.nix <&1 | grepQuietInverse "source path.*is uncacheable" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "source path.*cache hit" fi diff --git a/tests/functional/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh index b9045bf6b..b6c14fc26 100755 --- a/tests/functional/flakes/mercurial.sh +++ b/tests/functional/flakes/mercurial.sh @@ -27,9 +27,9 @@ nix build -o "$TEST_ROOT/result" "hg+file://$flake2Dir" (! nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revision) -nix eval "hg+file://$flake2Dir"#expr +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr -nix eval "hg+file://$flake2Dir"#expr +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr (! nix eval "hg+file://$flake2Dir"#expr --no-allow-dirty) diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh index f5e12cd01..7e55aca20 100644 --- a/tests/functional/flakes/non-flake-inputs.sh +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -72,7 +72,7 @@ nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file nix registry add --registry "$registry" flake3 "git+file://$flake3Dir" -nix build -o "$TEST_ROOT/result" flake3#fnord +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord [[ $(cat "$TEST_ROOT/result") = FNORD ]] # Check whether flake input fetching is lazy: flake3#sth does not @@ -82,11 +82,11 @@ clearStore mv "$flake2Dir" "$flake2Dir.tmp" mv "$nonFlakeDir" "$nonFlakeDir.tmp" nix build -o "$TEST_ROOT/result" flake3#sth -(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) -(! nix build -o "$TEST_ROOT/result" flake3#fnord) +(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy) +(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord) mv "$flake2Dir.tmp" "$flake2Dir" mv "$nonFlakeDir.tmp" "$nonFlakeDir" -nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore git -C "$flake3Dir" checkout -b removeXyzzy diff --git a/tests/functional/flakes/relative-paths-lockfile.sh b/tests/functional/flakes/relative-paths-lockfile.sh index d91aedd16..662c9329c 100644 --- a/tests/functional/flakes/relative-paths-lockfile.sh +++ b/tests/functional/flakes/relative-paths-lockfile.sh @@ -4,6 +4,8 @@ source ./common.sh requireGit +unset _NIX_TEST_BARF_ON_UNCACHEABLE + # Test a "vendored" subflake dependency. This is a relative path flake # which doesn't reference the root flake and has its own lock file. # From 1eab4236d482a9bfaf0042377928a300d8ac3f69 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 22:15:52 +0000 Subject: [PATCH 58/73] Prepare release v3.6.6 From 97af07180c1a2841de37c0bda9c33b37be9dad3a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 22:15:55 +0000 Subject: [PATCH 59/73] Set .version-determinate to 3.6.6 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index d15b8b06f..4f2c1d15f 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.5 +3.6.6 From f9b88e3229ee36e6f07e6277d4859e6d09f2693c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 22:16:00 +0000 Subject: [PATCH 60/73] Generate release notes for 3.6.6 --- doc/manual/source/SUMMARY.md.in | 1 + .../release-notes-determinate/changes.md | 22 ++++++++++++++++++- .../release-notes-determinate/rl-3.6.6.md | 17 ++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.6.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 374aacb59..dd3218d2f 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md) - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 30a68f6e9..a86b16a70 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.5. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.6. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -72,3 +72,23 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + + + +* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) + +* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) + +* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) + +* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) + +* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) + +* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) + +* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) + +* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) + +* Release v3.6.5 by @github-actions in [DeterminateSystems/nix-src#119](https://github.com/DeterminateSystems/nix-src/pull/119) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.6.md b/doc/manual/source/release-notes-determinate/rl-3.6.6.md new file mode 100644 index 000000000..55e903d7c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.6.md @@ -0,0 +1,17 @@ +# Release 3.6.6 (2025-06-17) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) +* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) +* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) +* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) +* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) +* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) +* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) +* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) +* Release v3.6.5 by @github-actions in [DeterminateSystems/nix-src#119](https://github.com/DeterminateSystems/nix-src/pull/119) + + +**Full Changelog**: [v3.6.3...v3.6.6](https://github.com/DeterminateSystems/nix-src/compare/v3.6.3...v3.6.6) From cd0128796a9462beb3c9db13db6409c9aa491fd4 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 17 Jun 2025 18:21:01 -0400 Subject: [PATCH 61/73] Apply suggestions from code review --- .../release-notes-determinate/changes.md | 18 ------------------ .../release-notes-determinate/rl-3.6.6.md | 12 +----------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index a86b16a70..6f27f7f6b 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -74,21 +74,3 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) - -* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) - -* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) - -* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) - -* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) - -* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) - -* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) - -* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) - -* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) - -* Release v3.6.5 by @github-actions in [DeterminateSystems/nix-src#119](https://github.com/DeterminateSystems/nix-src/pull/119) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.6.md b/doc/manual/source/release-notes-determinate/rl-3.6.6.md index 55e903d7c..bf4e3690a 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.6.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.6.md @@ -3,15 +3,5 @@ * Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). ## What's Changed -* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) -* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) -* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) -* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) -* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) -* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) -* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) -* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) -* Release v3.6.5 by @github-actions in [DeterminateSystems/nix-src#119](https://github.com/DeterminateSystems/nix-src/pull/119) - -**Full Changelog**: [v3.6.3...v3.6.6](https://github.com/DeterminateSystems/nix-src/compare/v3.6.3...v3.6.6) +* No-op release on the nix-src side, due to a regression on nix-darwin in determinate-nixd. From 86fbaf3b14f5c18fa35a015d958149c06575a0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= Date: Wed, 18 Jun 2025 10:05:02 +0200 Subject: [PATCH 62/73] tests: fixup with jq-1.8.0 (cherry picked from commit 77f6b6532f582a9db2bd6317f4fd272c32a05c7d) --- tests/functional/flakes/flakes.sh | 2 +- tests/functional/flakes/relative-paths.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index e8b051198..ce695a6cb 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -160,7 +160,7 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] -[[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] # Rerunning the build should not change the lockfile. nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 9d31da0ad..7a76bee1b 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -69,7 +69,7 @@ git -C "$rootFlake" add flake.nix sub2/flake.nix git -C "$rootFlake" add sub2/flake.lock [[ $(nix eval "$subflake2#y") = 15 ]] -[[ $(jq --indent 0 . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" "$subflake2/flake.lock") From 2f6c758d3d9452271948740b1ca2a4cdab9643c6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jun 2025 18:06:24 +0200 Subject: [PATCH 63/73] Revert "Drop magic-nix-cache" This reverts commit 9cc8be26747a0206613421a1ba1c3b1f54212e8b since magic-nix-cache works again (thanks @jchv). (cherry picked from commit 9b57573baea5abd242c5f62f537c7582c0097c3b) --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb70fae87..29cb33f56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: extra_nix_config: | sandbox = true max-jobs = 1 + - uses: DeterminateSystems/magic-nix-cache-action@main # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 @@ -133,6 +134,7 @@ jobs: - uses: cachix/install-nix-action@v31 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install + - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz @@ -174,6 +176,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main - run: | nix build -L \ .#hydraJobs.tests.functional_user \ @@ -199,4 +202,5 @@ jobs: repository: NixOS/flake-regressions-data path: flake-regressions/tests - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh From 5d2986d3c52b49ea82f9e2ea4d9a86929b55121a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= Date: Wed, 18 Jun 2025 10:05:02 +0200 Subject: [PATCH 64/73] tests: fixup with jq-1.8.0 --- tests/functional/flakes/flakes.sh | 4 ++-- tests/functional/flakes/relative-paths.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 261d65d69..a433cf71f 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -163,11 +163,11 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] -[[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] if [[ $(nix config show lazy-trees) = true ]]; then # Test that `lazy-locks` causes NAR hashes to be omitted from the lock file. nix flake update --flake "$flake2Dir" --commit-lock-file --lazy-locks - [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] + [[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] fi # Rerunning the build should not change the lockfile. diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 9d31da0ad..7a76bee1b 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -69,7 +69,7 @@ git -C "$rootFlake" add flake.nix sub2/flake.nix git -C "$rootFlake" add sub2/flake.lock [[ $(nix eval "$subflake2#y") = 15 ]] -[[ $(jq --indent 0 . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" "$subflake2/flake.lock") From 833406121cb65c42de8e0c4fad62be140b1b7978 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Jun 2025 19:14:31 +0200 Subject: [PATCH 65/73] Make the repl test more robust Seen in https://github.com/DeterminateSystems/nix-src/actions/runs/15590867877/job/43909540271: nix-functional-tests> grep: repl_output: No such file or directory nix-functional-tests> +(repl.sh:174) cat repl_output This is because there is a small possibility that the `nix repl` child process hasn't created `repl_output` yet. So make sure it exists. (cherry picked from commit 9eb46e9cc030016b1f4a073474a836bac1de3615) --- tests/functional/repl.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 762636e44..82a932e2b 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -163,7 +163,8 @@ foo + baz # - Re-eval it # - Check that the result has changed mkfifo repl_fifo -nix repl ./flake --experimental-features 'flakes' < repl_fifo > repl_output 2>&1 & +touch repl_output +nix repl ./flake --experimental-features 'flakes' < repl_fifo >> repl_output 2>&1 & repl_pid=$! exec 3>repl_fifo # Open fifo for writing echo "changingThing" >&3 From 58a878d846ffadd016595e9ed9d07fceac2199a1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 13:18:05 +0000 Subject: [PATCH 66/73] Prepare release v3.6.7 From ff05659f8bc5c84568e0ead4dbee1a8eb9705ee2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 13:18:08 +0000 Subject: [PATCH 67/73] Set .version-determinate to 3.6.7 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 4f2c1d15f..5b3413147 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.6 +3.6.7 From 098be10e285b05e6b04d3d7feb14270a9daefba1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 13:18:13 +0000 Subject: [PATCH 68/73] Generate release notes for 3.6.7 --- doc/manual/source/SUMMARY.md.in | 1 + .../source/release-notes-determinate/changes.md | 10 +++++++++- .../source/release-notes-determinate/rl-3.6.7.md | 11 +++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.7.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index dd3218d2f..b4458fc8c 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.7 (2025-06-24)](release-notes-determinate/rl-3.6.7.md) - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md) - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 6f27f7f6b..f3183883c 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.6. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.7. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -74,3 +74,11 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + + + +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) + +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) + +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.7.md b/doc/manual/source/release-notes-determinate/rl-3.6.7.md new file mode 100644 index 000000000..6ce425217 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.7.md @@ -0,0 +1,11 @@ +# Release 3.6.7 (2025-06-24) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) + + +**Full Changelog**: [v3.6.6...v3.6.7](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.7) From 731b63032161d3712ae26825230048dc5875eef4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Jun 2025 16:20:34 +0200 Subject: [PATCH 69/73] Fixes for GHSA-g948-229j-48j3 Squashed commit of the following: commit 04fff3a637d455cbb1d75937a235950e43008db9 Author: Eelco Dolstra Date: Thu Jun 12 12:30:32 2025 +0200 Chown structured attr files safely commit 5417ad445e414c649d0cfc71a05661c7bf8f3ef5 Author: Eelco Dolstra Date: Thu Jun 12 12:14:04 2025 +0200 Replace 'bool sync' with an enum for clarity And drop writeFileAndSync(). commit 7ae0141f328d8e8e1094be24665789c05f974ba6 Author: Eelco Dolstra Date: Thu Jun 12 11:35:28 2025 +0200 Drop guessOrInventPathFromFD() No need to do hacky stuff like that when we already know the original path. commit 45b05098bd019da7c57cd4227a89bfd0fa65bb08 Author: Eelco Dolstra Date: Thu Jun 12 11:15:58 2025 +0200 Tweak comment commit 0af15b31209d1b7ec8addfae9a1a6b60d8f35848 Author: Raito Bezarius Date: Thu Mar 27 12:22:26 2025 +0100 libstore: ensure that temporary directory is always 0o000 before deletion In the case the deletion fails, we should ensure that the temporary directory cannot be used for nefarious purposes. Change-Id: I498a2dd0999a74195d13642f44a5de1e69d46120 Signed-off-by: Raito Bezarius commit 2c20fa37b15cfa03ac6a1a6a47cdb2ed66c0827e Author: Raito Bezarius Date: Wed Mar 26 12:42:55 2025 +0100 libutil: ensure that `_deletePath` does NOT use absolute paths with dirfds When calling `_deletePath` with a parent file descriptor, `openat` is made effective by using relative paths to the directory file descriptor. To avoid the problem, the signature is changed to resist misuse with an assert in the prologue of the function. Change-Id: I6b3fc766bad2afe54dc27d47d1df3873e188de96 Signed-off-by: Raito Bezarius commit d3c370bbcae48bb825ce19fd0f73bb4eefd2c9ea Author: Raito Bezarius Date: Wed Mar 26 01:07:47 2025 +0100 libstore: ensure that `passAsFile` is created in the original temp dir This ensures that `passAsFile` data is created inside the expected temporary build directory by `openat()` from the parent directory file descriptor. This avoids a TOCTOU which is part of the attack chain of CVE-????. Change-Id: Ie5273446c4a19403088d0389ae8e3f473af8879a Signed-off-by: Raito Bezarius commit 45d3598724f932d024ef6bc2ffb00c1bb90e6018 Author: Raito Bezarius Date: Wed Mar 26 01:06:03 2025 +0100 libutil: writeFile variant for file descriptors `writeFile` lose its `sync` boolean flag to make things simpler. A new `writeFileAndSync` function is created and all call sites are converted to it. Change-Id: Ib871a5283a9c047db1e4fe48a241506e4aab9192 Signed-off-by: Raito Bezarius commit 732bd9b98cabf4aaf95a01fd318923de303f9996 Author: Raito Bezarius Date: Wed Mar 26 01:05:34 2025 +0100 libstore: chown to builder variant for file descriptors We use it immediately for the build temporary directory. Change-Id: I180193c63a2b98721f5fb8e542c4e39c099bb947 Signed-off-by: Raito Bezarius commit 962c65f8dcd5570dd92c72370a862c7b38942e0d Author: Raito Bezarius Date: Wed Mar 26 01:04:59 2025 +0100 libstore: open build directory as a dirfd as well We now keep around a proper AutoCloseFD around the temporary directory which we plan to use for openat operations and avoiding the build directory being swapped out while we are doing something else. Change-Id: I18d387b0f123ebf2d20c6405cd47ebadc5505f2a Signed-off-by: Raito Bezarius commit c9b42462b75b5a37ee6564c2b53cff186c8323da Author: Raito Bezarius Date: Wed Mar 26 01:04:12 2025 +0100 libutil: guess or invent a path from file descriptors This is useful for certain error recovery paths (no pun intended) that does not thread through the original path name. Change-Id: I2d800740cb4f9912e64c923120d3f977c58ccb7e Signed-off-by: Raito Bezarius --- src/libstore/local-store.cc | 4 +- src/libstore/unix/build/derivation-builder.cc | 66 ++++++++++++++++--- src/libutil/file-content-address.cc | 2 +- src/libutil/file-system.cc | 53 ++++++++------- src/libutil/include/nix/util/file-system.hh | 14 ++-- 5 files changed, 101 insertions(+), 38 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 76fadba86..1ab3ed13a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -247,7 +247,7 @@ LocalStore::LocalStore(ref config) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(*state, true); - writeFile(schemaPath, fmt("%1%", curSchema), 0666, true); + writeFile(schemaPath, fmt("%1%", curSchema), 0666, FsSync::Yes); } else if (curSchema < nixSchemaVersion) { @@ -298,7 +298,7 @@ LocalStore::LocalStore(ref config) txn.commit(); } - writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, FsSync::Yes); lockFile(globalLock.get(), ltRead, true); } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index e84e2db6e..43dfe1832 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -129,6 +129,11 @@ private: */ Path topTmpDir; + /** + * The file descriptor of the temporary directory. + */ + AutoCloseFD tmpDirFd; + /** * The path of the temporary directory in the sandbox. */ @@ -325,9 +330,24 @@ private: /** * Make a file owned by the builder. + * + * SAFETY: this function is prone to TOCTOU as it receives a path and not a descriptor. + * It's only safe to call in a child of a directory only visible to the owner. */ void chownToBuilder(const Path & path); + /** + * Make a file owned by the builder addressed by its file descriptor. + */ + void chownToBuilder(int fd, const Path & path); + + /** + * Create a file in `tmpDir` owned by the builder. + */ + void writeBuilderFile( + const std::string & name, + std::string_view contents); + /** * Run the builder's process. */ @@ -900,7 +920,14 @@ void DerivationBuilderImpl::startBuilder() } else { tmpDir = topTmpDir; } - chownToBuilder(tmpDir); + + /* The TOCTOU between the previous mkdir call and this open call is unavoidable due to + POSIX semantics.*/ + tmpDirFd = AutoCloseFD{open(tmpDir.c_str(), O_RDONLY | O_NOFOLLOW | O_DIRECTORY)}; + if (!tmpDirFd) + throw SysError("failed to open the build temporary directory descriptor '%1%'", tmpDir); + + chownToBuilder(tmpDirFd.get(), tmpDir); for (auto & [outputName, status] : initialOutputs) { /* Set scratch path we'll actually use during the build. @@ -1485,9 +1512,7 @@ void DerivationBuilderImpl::initTmpDir() } else { auto hash = hashString(HashAlgorithm::SHA256, i.first); std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); + writeBuilderFile(fn, rewriteStrings(i.second, inputRewrites)); env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } } @@ -1596,11 +1621,9 @@ void DerivationBuilderImpl::writeStructuredAttrs() auto jsonSh = StructuredAttrs::writeShell(json); - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); + writeBuilderFile(".attrs.sh", rewriteStrings(jsonSh, inputRewrites)); env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); + writeBuilderFile(".attrs.json", rewriteStrings(json.dump(), inputRewrites)); env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; } } @@ -1854,6 +1877,24 @@ void setupSeccomp() #endif } +void DerivationBuilderImpl::chownToBuilder(int fd, const Path & path) +{ + if (!buildUser) return; + if (fchown(fd, buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError("cannot change ownership of file '%1%'", path); +} + +void DerivationBuilderImpl::writeBuilderFile( + const std::string & name, + std::string_view contents) +{ + auto path = std::filesystem::path(tmpDir) / name; + AutoCloseFD fd{openat(tmpDirFd.get(), name.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666)}; + if (!fd) + throw SysError("creating file %s", path); + writeFile(fd, path, contents); + chownToBuilder(fd.get(), path); +} void DerivationBuilderImpl::runChild() { @@ -3065,6 +3106,15 @@ void DerivationBuilderImpl::checkOutputs(const std::mapd_name; if (childName == "." || childName == "..") continue; - _deletePath(dirfd(dir.get()), path + "/" + childName, bytesFreed, ex); + _deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex); } if (errno) throw SysError("reading directory %1%", path); } @@ -497,14 +505,13 @@ static void _deletePath(Descriptor parentfd, const std::filesystem::path & path, static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed) { - Path dir = dirOf(path.string()); - if (dir == "") - dir = "/"; + assert(path.is_absolute()); + assert(path.parent_path() != path); - AutoCloseFD dirfd = toDescriptor(open(dir.c_str(), O_RDONLY)); + AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY)); if (!dirfd) { if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); + throw SysError("opening directory %s", path.parent_path()); } std::exception_ptr ex; diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index b8fa4cfa0..a9a6e43bf 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -175,21 +175,27 @@ std::string readFile(const Path & path); std::string readFile(const std::filesystem::path & path); void readFile(const Path & path, Sink & sink, bool memory_map = true); +enum struct FsSync { Yes, No }; + /** * Write a string to a file. */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); -static inline void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, bool sync = false) +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No); + +static inline void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No) { return writeFile(path.string(), s, mode, sync); } -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); -static inline void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, bool sync = false) +void writeFile(const Path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No); + +static inline void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No) { return writeFile(path.string(), source, mode, sync); } +void writeFile(AutoCloseFD & fd, const Path & origPath, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No); + /** * Flush a path's parent directory to disk. */ From 404d82419405c31dd27a2fb27b387f0497fb26c4 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 24 Jun 2025 09:29:07 -0400 Subject: [PATCH 70/73] Update changelogs --- .../source/release-notes-determinate/changes.md | 6 ------ .../source/release-notes-determinate/rl-3.6.7.md | 12 +++++++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index f3183883c..95374dcb6 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -76,9 +76,3 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix - -* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) - -* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) - -* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.7.md b/doc/manual/source/release-notes-determinate/rl-3.6.7.md index 6ce425217..197587f1b 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.7.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.7.md @@ -3,9 +3,15 @@ * Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). ## What's Changed -* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) -* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) -* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) + +### Security contents + +* Patched against GHSA-g948-229j-48j3 + +### Lazy trees: + +* Lazy trees now produces `flake.lock` files with NAR hashes unless `lazy-locks` is set to `true` by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Improved caching with lazy-trees when using --impure, with enhanced testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) **Full Changelog**: [v3.6.6...v3.6.7](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.7) From c2baff64f7f9d0c6a5135acfdca483c9c9f41712 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 03:32:32 +0000 Subject: [PATCH 71/73] Prepare release v3.6.8 From fd8b3270602406aa492af4d32ee47ccc3a9d6f03 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 03:32:35 +0000 Subject: [PATCH 72/73] Set .version-determinate to 3.6.8 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 5b3413147..424e1794d 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.7 +3.6.8 From 4ba66db9f66297412be45385d6ed1654fda0408f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 03:32:40 +0000 Subject: [PATCH 73/73] Generate release notes for 3.6.8 --- doc/manual/source/SUMMARY.md.in | 1 + .../source/release-notes-determinate/changes.md | 12 +++++++++++- .../source/release-notes-determinate/rl-3.6.8.md | 12 ++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.8.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index b4458fc8c..ebcb7b956 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.8 (2025-06-25)](release-notes-determinate/rl-3.6.8.md) - [Release 3.6.7 (2025-06-24)](release-notes-determinate/rl-3.6.7.md) - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md) - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 95374dcb6..3a38378e0 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.7. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.8. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -76,3 +76,13 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix + + + +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) + +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) + +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) + +* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.8.md b/doc/manual/source/release-notes-determinate/rl-3.6.8.md new file mode 100644 index 000000000..c4b4b96c9 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.8.md @@ -0,0 +1,12 @@ +# Release 3.6.8 (2025-06-25) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) +* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126) + + +**Full Changelog**: [v3.6.6...v3.6.8](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.8)