From c4813b8cbc3f8559c60d1a387036c64e2649734c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 May 2025 14:48:51 +0200 Subject: [PATCH 01/31] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index f01356823..69886179f 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.29.0 +2.29.1 From acc3cd460d8968b6a90bf8f9076280b4aa7a6961 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 23 May 2025 23:33:59 +0200 Subject: [PATCH 02/31] Don't use 'callback' object that we may have moved out of (cherry picked from commit fa6e10ea6a87127ae813a708ccc97e708982f93f) --- src/libstore/http-binary-cache-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 2b591dda9..e44d146b9 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -176,13 +176,13 @@ protected: void getFile(const std::string & path, Callback> callback) noexcept override { + auto callbackPtr = std::make_shared(std::move(callback)); + try { checkEnabled(); auto request(makeRequest(path)); - auto callbackPtr = std::make_shared(std::move(callback)); - getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future result) { try { @@ -198,7 +198,7 @@ protected: }}); } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); return; } } From 8ca4d2ef08e8e19c893ec186cfe8aeee91e94041 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 24 May 2025 00:14:32 +0200 Subject: [PATCH 03/31] fromStructuredAttrs(): Don't crash if exportReferencesGraph is a string Fixes error: [json.exception.type_error.302] type must be array, but is string and other crashes. Fixes #13254. (cherry picked from commit d877b0c0cc4795d17d10b9b9039f2de828152c55) --- src/libstore/derivation-options.cc | 9 +++++++-- src/libstore/misc.cc | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index e031f8447..f6bac2868 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -211,8 +211,13 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); if (!e || !e->is_object()) return ret; - for (auto & [key, storePathsJson] : getObject(*e)) { - ret.insert_or_assign(key, storePathsJson); + for (auto & [key, value] : getObject(*e)) { + if (value.is_array()) + ret.insert_or_assign(key, value); + else if (value.is_string()) + ret.insert_or_assign(key, StringSet{value}); + else + throw Error("'exportReferencesGraph' value is not an array or a string"); } } else { auto s = getOr(env, "exportReferencesGraph", ""); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 967c91d72..dabae647f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -225,6 +225,8 @@ void Store::queryMissing(const std::vector & targets, auto parsedDrv = StructuredAttrs::tryParse(drv->env); DerivationOptions drvOptions; try { + // FIXME: this is a lot of work just to get the value + // of `allowSubstitutes`. drvOptions = DerivationOptions::fromStructuredAttrs( drv->env, parsedDrv ? &*parsedDrv : nullptr); From d068b82c2f27d4105f54b43dde34a67dbd7d6db9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 24 May 2025 00:40:06 +0200 Subject: [PATCH 04/31] Add test (cherry picked from commit c66eb9cef77c3462d0324b258d0c5e0b8e4f4e7f) --- tests/functional/structured-attrs-shell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/structured-attrs-shell.nix b/tests/functional/structured-attrs-shell.nix index a819e39cd..e9b9f1e39 100644 --- a/tests/functional/structured-attrs-shell.nix +++ b/tests/functional/structured-attrs-shell.nix @@ -21,7 +21,7 @@ mkDerivation { "b" "c" ]; - exportReferencesGraph.refs = [ dep ]; + exportReferencesGraph.refs = dep; buildCommand = '' touch ''${outputs[out]}; touch ''${outputs[dev]} ''; From cfba4b3bf41ed01a30a98e6bc5db96c909d2e73d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2025 23:30:16 +0200 Subject: [PATCH 05/31] Drop magic-nix-cache This no longer works, see https://determinate.systems/posts/magic-nix-cache-free-tier-eol/. (cherry picked from commit 9cc8be26747a0206613421a1ba1c3b1f54212e8b) --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29cb33f56..fb70fae87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,6 @@ jobs: extra_nix_config: | sandbox = true max-jobs = 1 - - uses: DeterminateSystems/magic-nix-cache-action@main # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 @@ -134,7 +133,6 @@ jobs: - uses: cachix/install-nix-action@v31 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz @@ -176,7 +174,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - run: | nix build -L \ .#hydraJobs.tests.functional_user \ @@ -202,5 +199,4 @@ jobs: repository: NixOS/flake-regressions-data path: flake-regressions/tests - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh From a2567f6d7ae9bcd7771a8790c0a9196e90ce097d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 29 May 2025 19:35:12 +0000 Subject: [PATCH 06/31] Clear `displayPrefix` in `makeEmptySourceAccessor` Judging by the comment for `makeEmptySourceAccessor` the prefix has to be empty: > Return a source accessor that contains only an empty root directory. Fixes #13295. (cherry picked from commit fba1bb0c137036adc5127afe4183f45ab3dde61d) --- src/libutil/memory-source-accessor.cc | 4 ++++ tests/functional/pure-eval.sh | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index 7764ff946..5612c9454 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -187,6 +187,10 @@ void MemorySink::createSymlink(const CanonPath & path, const std::string & targe ref makeEmptySourceAccessor() { static auto empty = make_ref().cast(); + /* Don't forget to clear the display prefix, as the default constructed + SourceAccessor has the «unknown» prefix. Since this accessor is supposed + to mimic an empty root directory the prefix needs to be empty. */ + empty->setPathDisplay(""); return empty; } diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh index 250381099..45a65f9ab 100755 --- a/tests/functional/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -34,3 +34,15 @@ rm -rf $TEST_ROOT/eval-out (! nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ "." = "bla"; }') (! nix eval --expr '~/foo') + +expectStderr 0 nix eval --expr "/some/absolute/path" \ + | grepQuiet "/some/absolute/path" + +expectStderr 0 nix eval --expr "/some/absolute/path" --impure \ + | grepQuiet "/some/absolute/path" + +expectStderr 0 nix eval --expr "some/relative/path" \ + | grepQuiet "$PWD/some/relative/path" + +expectStderr 0 nix eval --expr "some/relative/path" --impure \ + | grepQuiet "$PWD/some/relative/path" From fcdffffa37ab99b15490bb633698ee9fe03e7056 Mon Sep 17 00:00:00 2001 From: Seth Flynn Date: Tue, 27 May 2025 22:20:53 -0400 Subject: [PATCH 07/31] lockFlake(): Allow registry lookups for overridden inputs Fixes #13144 (cherry picked from commit d0a23238294198f6702e13d117f75af89dbeac62) --- src/libflake/flake.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 987c9f610..d5dafff7a 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -570,7 +570,7 @@ LockedFlake lockFlake( /* Get the input flake, resolve 'path:./...' flakerefs relative to the parent flake. */ - auto getInputFlake = [&](const FlakeRef & ref) + auto getInputFlake = [&](const FlakeRef & ref, const fetchers::UseRegistries useRegistries) { if (auto resolvedPath = resolveRelativePath()) { return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); @@ -578,7 +578,7 @@ LockedFlake lockFlake( return getFlake( state, ref, - useRegistriesInputs, + useRegistries, inputAttrPath); } }; @@ -660,7 +660,7 @@ LockedFlake lockFlake( } if (mustRefetch) { - auto inputFlake = getInputFlake(oldLock->lockedRef); + auto inputFlake = getInputFlake(oldLock->lockedRef, useRegistriesInputs); nodePaths.emplace(childNode, inputFlake.path.parent()); computeLocks(inputFlake.inputs, childNode, inputAttrPath, oldLock, followsPrefix, inputFlake.path, false); @@ -685,10 +685,11 @@ LockedFlake lockFlake( nuked the next time we update the lock file. That is, overrides are sticky unless you use --no-write-lock-file. */ - auto ref = (input2.ref && explicitCliOverrides.contains(inputAttrPath)) ? *input2.ref : *input.ref; + auto inputIsOverride = explicitCliOverrides.contains(inputAttrPath); + auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; if (input.isFlake) { - auto inputFlake = getInputFlake(*input.ref); + auto inputFlake = getInputFlake(*input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); auto childNode = make_ref( inputFlake.lockedRef, From a572c9433330ea307a3b5c83db2357d6ca55f8e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2025 20:19:19 +0200 Subject: [PATCH 08/31] Fix deep overrides An override like inputs.foo.inputs.bar.inputs.nixpkgs.follows = "nixpkgs"; implicitly set `inputs.foo.inputs.bar` to `flake:bar`, which led to an unexpected error like error: cannot find flake 'flake:bar' in the flake registries We now no longer create a parent override (like for `foo.bar` in the example above) if it doesn't set an explicit ref or follows attribute. We only recursively apply its child overrides. Fixes https://github.com/NixOS/nix/issues/8325, https://github.com/DeterminateSystems/nix-src/issues/95, https://github.com/NixOS/nix/issues/12083, https://github.com/NixOS/nix/issues/5790. --- src/libflake/flake.cc | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 31328abde..5570422a3 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -85,7 +85,6 @@ static void parseFlakeInputAttr( static FlakeInput parseFlakeInput( EvalState & state, - std::string_view inputName, Value * value, const PosIdx pos, const InputAttrPath & lockRootAttrPath, @@ -155,9 +154,6 @@ static FlakeInput parseFlakeInput( input.ref = parseFlakeRef(state.fetchSettings, *url, {}, true, input.isFlake, true); } - if (!input.follows && !input.ref) - input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(inputName)}}); - return input; } @@ -185,7 +181,6 @@ static std::pair, fetchers::Attrs> parseFlakeInput } else { inputs.emplace(inputName, parseFlakeInput(state, - inputName, inputAttr.value, inputAttr.pos, lockRootAttrPath, @@ -467,18 +462,27 @@ LockedFlake lockFlake( /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ - for (auto & [id, input] : flakeInputs) { + std::function addOverrides; + addOverrides = [&](const FlakeInput & input, const InputAttrPath & prefix) + { for (auto & [idOverride, inputOverride] : input.overrides) { - auto inputAttrPath(inputAttrPathPrefix); - inputAttrPath.push_back(id); + auto inputAttrPath(prefix); inputAttrPath.push_back(idOverride); - overrides.emplace(inputAttrPath, - OverrideTarget { - .input = inputOverride, - .sourcePath = sourcePath, - .parentInputAttrPath = inputAttrPathPrefix - }); + if (inputOverride.ref || inputOverride.follows) + overrides.emplace(inputAttrPath, + OverrideTarget { + .input = inputOverride, + .sourcePath = sourcePath, + .parentInputAttrPath = inputAttrPathPrefix + }); + addOverrides(inputOverride, inputAttrPath); } + }; + + for (auto & [id, input] : flakeInputs) { + auto inputAttrPath(inputAttrPathPrefix); + inputAttrPath.push_back(id); + addOverrides(input, inputAttrPath); } /* Check whether this input has overrides for a @@ -534,7 +538,8 @@ LockedFlake lockFlake( continue; } - assert(input.ref); + if (!input.ref) + input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(id)}}); auto overridenParentPath = input.ref->input.isRelative() From 9a18a11d7d8bd4c0b606cd16452eba6819464a6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2025 20:33:28 +0200 Subject: [PATCH 09/31] Add tests for deep overrides Taken from https://github.com/NixOS/nix/pull/6621. Co-authored-by: Sebastian Ullrich --- tests/functional/flakes/follow-paths.sh | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 25f26137b..9ee8e738e 100755 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -359,3 +359,63 @@ rm "$flakeFollowsCustomUrlA"/flake.lock json=$(nix flake metadata "$flakeFollowsCustomUrlA" --override-input B/C "$flakeFollowsCustomUrlD" --json) echo "$json" | jq .locks.nodes.C.original [[ $(echo "$json" | jq -r .locks.nodes.C.original.path) = './flakeC' ]] + +# Test deep overrides, e.g. `inputs.B.inputs.C.inputs.D.follows = ...`. + +cat < $flakeFollowsD/flake.nix +{ outputs = _: {}; } +EOF +cat < $flakeFollowsC/flake.nix +{ + inputs.D.url = "path:nosuchflake"; + outputs = _: {}; +} +EOF +cat < $flakeFollowsB/flake.nix +{ + inputs.C.url = "path:$flakeFollowsC"; + outputs = _: {}; +} +EOF +cat < $flakeFollowsA/flake.nix +{ + inputs.B.url = "path:$flakeFollowsB"; + inputs.D.url = "path:$flakeFollowsD"; + inputs.B.inputs.C.inputs.D.follows = "D"; + outputs = _: {}; +} +EOF + +nix flake lock $flakeFollowsA + +[[ $(jq -c .nodes.C.inputs.D $flakeFollowsA/flake.lock) = '["D"]' ]] + +# Test overlapping flake follows: B has D follow C/D, while A has B/C follow C + +cat < $flakeFollowsC/flake.nix +{ + inputs.D.url = "path:$flakeFollowsD"; + outputs = _: {}; +} +EOF +cat < $flakeFollowsB/flake.nix +{ + inputs.C.url = "path:nosuchflake"; + inputs.D.url = "path:nosuchflake"; + inputs.D.follows = "C/D"; + outputs = _: {}; +} +EOF +cat < $flakeFollowsA/flake.nix +{ + inputs.B.url = "path:$flakeFollowsB"; + inputs.C.url = "path:$flakeFollowsC"; + inputs.B.inputs.C.follows = "C"; + outputs = _: {}; +} +EOF + +# bug was not triggered without recreating the lockfile +nix flake lock $flakeFollowsA --recreate-lock-file + +[[ $(jq -c .nodes.B.inputs.D $flakeFollowsA/flake.lock) = '["B","C","D"]' ]] From 6999183956d360c1b91251f3628e7377f2751009 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2025 20:38:51 +0200 Subject: [PATCH 10/31] Don't allow flake inputs to have both a flakeref and a follows Having both doesn't make sense so it's best to disallow it. If this causes issues we could turn into a warning. --- src/libflake/flake.cc | 3 +++ tests/functional/flakes/follow-paths.sh | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 5570422a3..d9f042953 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -154,6 +154,9 @@ static FlakeInput parseFlakeInput( input.ref = parseFlakeRef(state.fetchSettings, *url, {}, true, input.isFlake, true); } + if (input.ref && input.follows) + throw Error("flake input has both a flake reference and a follows attribute, at %s", state.positions[pos]); + return input; } diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 9ee8e738e..952aed040 100755 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -401,7 +401,6 @@ EOF cat < $flakeFollowsB/flake.nix { inputs.C.url = "path:nosuchflake"; - inputs.D.url = "path:nosuchflake"; inputs.D.follows = "C/D"; outputs = _: {}; } @@ -419,3 +418,15 @@ EOF nix flake lock $flakeFollowsA --recreate-lock-file [[ $(jq -c .nodes.B.inputs.D $flakeFollowsA/flake.lock) = '["B","C","D"]' ]] + +# Check that you can't have both a flakeref and a follows attribute on an input. +cat < $flakeFollowsB/flake.nix +{ + inputs.C.url = "path:nosuchflake"; + inputs.D.url = "path:nosuchflake"; + inputs.D.follows = "C/D"; + outputs = _: {}; +} +EOF + +expectStderr 1 nix flake lock $flakeFollowsA --recreate-lock-file | grepQuiet "flake input has both a flake reference and a follows attribute" From 802f58540618ced207c31822af8feb7f2e67853b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Jun 2025 16:56:13 +0200 Subject: [PATCH 11/31] Add lazy-locks setting This determines whether lock file entries omit a NAR hash. They're included by default to make lazy trees compatible with older clients. --- src/libexpr/include/nix/expr/eval-settings.hh | 13 +++++++++++++ src/libexpr/paths.cc | 2 +- tests/functional/flakes/flakes.sh | 9 +++++---- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index 782f5f9e1..cb4726837 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -262,6 +262,19 @@ struct EvalSettings : Config R"( If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily. )"}; + + // FIXME: this setting should really be in libflake, but it's + // currently needed in mountInput(). + Setting lazyLocks{ + this, + false, + "lazy-locks", + R"( + If enabled, Nix will only include NAR hashes in lock file entries if they're necessary to lock the input (i.e. when there is no other attribute that allows the content to be verified, like a Git revision). + This is not backward compatible with older versions of Nix. + If disabled, lock file entries will always contain a NAR hash. + )" + }; }; /** diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 65b8212e1..7bac317d9 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -92,7 +92,7 @@ StorePath EvalState::mountInput( storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); - if (requireLockable && (!settings.lazyTrees || !input.isLocked()) && !input.getNarHash()) + if (requireLockable && (!settings.lazyTrees || !settings.lazyLocks || !input.isLocked()) && !input.getNarHash()) input.attrs.insert_or_assign("narHash", getNarHash()->to_string(HashFormat::SRI, true)); if (originalInput.getNarHash() && *getNarHash() != *originalInput.getNarHash()) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index e335fe6f3..261d65d69 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -163,10 +163,11 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] -if [[ $(nix config show lazy-trees) = false ]]; then - [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] -else - [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +if [[ $(nix config show lazy-trees) = true ]]; then + # Test that `lazy-locks` causes NAR hashes to be omitted from the lock file. + nix flake update --flake "$flake2Dir" --commit-lock-file --lazy-locks + [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] fi # Rerunning the build should not change the lockfile. From b2c762cd2f204624cb1e6ee9df8495942db28742 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Jun 2025 21:03:58 +0200 Subject: [PATCH 12/31] Apply suggestions from code review Co-authored-by: Luc Perkins --- src/libexpr/include/nix/expr/eval-settings.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index cb4726837..9b7573b20 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -270,9 +270,9 @@ struct EvalSettings : Config false, "lazy-locks", R"( - If enabled, Nix will only include NAR hashes in lock file entries if they're necessary to lock the input (i.e. when there is no other attribute that allows the content to be verified, like a Git revision). + If enabled, Nix only includes NAR hashes in lock file entries if they're necessary to lock the input (i.e. when there is no other attribute that allows the content to be verified, like a Git revision). This is not backward compatible with older versions of Nix. - If disabled, lock file entries will always contain a NAR hash. + If disabled, lock file entries always contain a NAR hash. )" }; }; From 5329a45ade7fd94c180a538e0e7832fbff1220e7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 18:16:30 +0200 Subject: [PATCH 13/31] Git fetcher: Make dirty repos with no commits cacheable --- src/libfetchers/git.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 2825b72ab..4a00d4e34 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -860,7 +860,7 @@ struct GitInputScheme : InputScheme return makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); - if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { + if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) { /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; @@ -873,7 +873,7 @@ struct GitInputScheme : InputScheme writeString("deleted:", hashSink); writeString(file.abs(), hashSink); } - return makeFingerprint(*repoInfo.workdirInfo.headRev) + return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev)) + ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false); } return std::nullopt; From a175c67def61f68b8b1ac860ea7b0672badd1d6e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 19:34:28 +0200 Subject: [PATCH 14/31] Fix rootFS fingerprint in the impure case --- src/libutil/union-source-accessor.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index 9950f6049..69cf04c18 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -72,6 +72,18 @@ struct UnionSourceAccessor : SourceAccessor } return std::nullopt; } + + std::pair> getFingerprint(const CanonPath & path) override + { + if (fingerprint) + return {path, fingerprint}; + for (auto & accessor : accessors) { + auto [subpath, fingerprint] = accessor->getFingerprint(path); + if (fingerprint) + return {subpath, fingerprint}; + } + return {path, std::nullopt}; + } }; ref makeUnionSourceAccessor(std::vector> && accessors) From 3a4e6cadebacba2723306f2da7f9bdd12ccdf5ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 18:25:55 +0200 Subject: [PATCH 15/31] Improve regression testing for uncachable source paths These now throw an error by default in the test suite. --- src/libfetchers/fetch-to-store.cc | 8 ++++++-- tests/functional/flakes/common.sh | 2 ++ tests/functional/flakes/flake-in-submodule.sh | 6 +++--- tests/functional/flakes/flakes.sh | 1 - tests/functional/flakes/mercurial.sh | 4 ++-- tests/functional/flakes/non-flake-inputs.sh | 8 ++++---- tests/functional/flakes/relative-paths-lockfile.sh | 2 ++ 7 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 877e49c14..e6b9430a2 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -55,9 +55,13 @@ std::pair fetchToStore2( } debug("source path '%s' not in store", path); } - } else + } else { + static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1"; + if (barf) + throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter); // FIXME: could still provide in-memory caching keyed on `SourcePath`. - debug("source path '%s' is uncacheable (%d, %d)", path, (bool) filter, (bool) fingerprint); + debug("source path '%s' is uncacheable", path); + } Activity act(*logger, lvlChatty, actUnknown, fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path)); diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 422cab96c..77bc03060 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -2,6 +2,8 @@ source ../common.sh +export _NIX_TEST_BARF_ON_UNCACHEABLE=1 + # shellcheck disable=SC2034 # this variable is used by tests that source this file registry=$TEST_ROOT/registry.json diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index fe5acf26d..a7d86698d 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -62,8 +62,8 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule # Check that dirtying a submodule makes the entire thing dirty. [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]] echo '"foo"' > "$rootRepo"/submodule/sub.nix -[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] -[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval --json "$flakeref#sub" ) = '"foo"' ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] # Test that `nix flake metadata` parses `submodule` correctly. cat > "$rootRepo"/flake.nix <&1 | grepQuietInverse "source path.*is uncacheable" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "source path.*cache hit" fi diff --git a/tests/functional/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh index b9045bf6b..b6c14fc26 100755 --- a/tests/functional/flakes/mercurial.sh +++ b/tests/functional/flakes/mercurial.sh @@ -27,9 +27,9 @@ nix build -o "$TEST_ROOT/result" "hg+file://$flake2Dir" (! nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revision) -nix eval "hg+file://$flake2Dir"#expr +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr -nix eval "hg+file://$flake2Dir"#expr +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr (! nix eval "hg+file://$flake2Dir"#expr --no-allow-dirty) diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh index f5e12cd01..7e55aca20 100644 --- a/tests/functional/flakes/non-flake-inputs.sh +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -72,7 +72,7 @@ nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file nix registry add --registry "$registry" flake3 "git+file://$flake3Dir" -nix build -o "$TEST_ROOT/result" flake3#fnord +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord [[ $(cat "$TEST_ROOT/result") = FNORD ]] # Check whether flake input fetching is lazy: flake3#sth does not @@ -82,11 +82,11 @@ clearStore mv "$flake2Dir" "$flake2Dir.tmp" mv "$nonFlakeDir" "$nonFlakeDir.tmp" nix build -o "$TEST_ROOT/result" flake3#sth -(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) -(! nix build -o "$TEST_ROOT/result" flake3#fnord) +(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy) +(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord) mv "$flake2Dir.tmp" "$flake2Dir" mv "$nonFlakeDir.tmp" "$nonFlakeDir" -nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore git -C "$flake3Dir" checkout -b removeXyzzy diff --git a/tests/functional/flakes/relative-paths-lockfile.sh b/tests/functional/flakes/relative-paths-lockfile.sh index d91aedd16..662c9329c 100644 --- a/tests/functional/flakes/relative-paths-lockfile.sh +++ b/tests/functional/flakes/relative-paths-lockfile.sh @@ -4,6 +4,8 @@ source ./common.sh requireGit +unset _NIX_TEST_BARF_ON_UNCACHEABLE + # Test a "vendored" subflake dependency. This is a relative path flake # which doesn't reference the root flake and has its own lock file. # From 1eab4236d482a9bfaf0042377928a300d8ac3f69 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 22:15:52 +0000 Subject: [PATCH 16/31] Prepare release v3.6.6 From 97af07180c1a2841de37c0bda9c33b37be9dad3a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 22:15:55 +0000 Subject: [PATCH 17/31] Set .version-determinate to 3.6.6 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index d15b8b06f..4f2c1d15f 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.5 +3.6.6 From f9b88e3229ee36e6f07e6277d4859e6d09f2693c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 22:16:00 +0000 Subject: [PATCH 18/31] Generate release notes for 3.6.6 --- doc/manual/source/SUMMARY.md.in | 1 + .../release-notes-determinate/changes.md | 22 ++++++++++++++++++- .../release-notes-determinate/rl-3.6.6.md | 17 ++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.6.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 374aacb59..dd3218d2f 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md) - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 30a68f6e9..a86b16a70 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.5. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.6. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -72,3 +72,23 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + + + +* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) + +* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) + +* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) + +* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) + +* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) + +* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) + +* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) + +* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) + +* Release v3.6.5 by @github-actions in [DeterminateSystems/nix-src#119](https://github.com/DeterminateSystems/nix-src/pull/119) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.6.md b/doc/manual/source/release-notes-determinate/rl-3.6.6.md new file mode 100644 index 000000000..55e903d7c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.6.md @@ -0,0 +1,17 @@ +# Release 3.6.6 (2025-06-17) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) +* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) +* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) +* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) +* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) +* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) +* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) +* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) +* Release v3.6.5 by @github-actions in [DeterminateSystems/nix-src#119](https://github.com/DeterminateSystems/nix-src/pull/119) + + +**Full Changelog**: [v3.6.3...v3.6.6](https://github.com/DeterminateSystems/nix-src/compare/v3.6.3...v3.6.6) From cd0128796a9462beb3c9db13db6409c9aa491fd4 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 17 Jun 2025 18:21:01 -0400 Subject: [PATCH 19/31] Apply suggestions from code review --- .../release-notes-determinate/changes.md | 18 ------------------ .../release-notes-determinate/rl-3.6.6.md | 12 +----------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index a86b16a70..6f27f7f6b 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -74,21 +74,3 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) - -* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) - -* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) - -* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) - -* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) - -* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) - -* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) - -* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) - -* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) - -* Release v3.6.5 by @github-actions in [DeterminateSystems/nix-src#119](https://github.com/DeterminateSystems/nix-src/pull/119) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.6.md b/doc/manual/source/release-notes-determinate/rl-3.6.6.md index 55e903d7c..bf4e3690a 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.6.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.6.md @@ -3,15 +3,5 @@ * Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). ## What's Changed -* Release v3.6.4 by @github-actions in [DeterminateSystems/nix-src#109](https://github.com/DeterminateSystems/nix-src/pull/109) -* Fixup the complainy docs line by @grahamc in [DeterminateSystems/nix-src#111](https://github.com/DeterminateSystems/nix-src/pull/111) -* Move the actual vm tests / flake regressions into the generic build phase by @grahamc in [DeterminateSystems/nix-src#112](https://github.com/DeterminateSystems/nix-src/pull/112) -* Fix broken fetchToStore() caching by @edolstra in [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110) -* Parallelize the flake regression suite by @grahamc in [DeterminateSystems/nix-src#114](https://github.com/DeterminateSystems/nix-src/pull/114) -* Don't build fallback-paths if we didn't build aarch64-linux and x86 d… …arwin by @grahamc in [DeterminateSystems/nix-src#116](https://github.com/DeterminateSystems/nix-src/pull/116) -* Use GHA runners for VMs since they have KVM by @grahamc in [DeterminateSystems/nix-src#118](https://github.com/DeterminateSystems/nix-src/pull/118) -* fetchToStore() cache: Use content hashes instead of store paths by @edolstra in [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) -* Release v3.6.5 by @github-actions in [DeterminateSystems/nix-src#119](https://github.com/DeterminateSystems/nix-src/pull/119) - -**Full Changelog**: [v3.6.3...v3.6.6](https://github.com/DeterminateSystems/nix-src/compare/v3.6.3...v3.6.6) +* No-op release on the nix-src side, due to a regression on nix-darwin in determinate-nixd. From 86fbaf3b14f5c18fa35a015d958149c06575a0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= Date: Wed, 18 Jun 2025 10:05:02 +0200 Subject: [PATCH 20/31] tests: fixup with jq-1.8.0 (cherry picked from commit 77f6b6532f582a9db2bd6317f4fd272c32a05c7d) --- tests/functional/flakes/flakes.sh | 2 +- tests/functional/flakes/relative-paths.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index e8b051198..ce695a6cb 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -160,7 +160,7 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] -[[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] # Rerunning the build should not change the lockfile. nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 9d31da0ad..7a76bee1b 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -69,7 +69,7 @@ git -C "$rootFlake" add flake.nix sub2/flake.nix git -C "$rootFlake" add sub2/flake.lock [[ $(nix eval "$subflake2#y") = 15 ]] -[[ $(jq --indent 0 . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" "$subflake2/flake.lock") From 2f6c758d3d9452271948740b1ca2a4cdab9643c6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jun 2025 18:06:24 +0200 Subject: [PATCH 21/31] Revert "Drop magic-nix-cache" This reverts commit 9cc8be26747a0206613421a1ba1c3b1f54212e8b since magic-nix-cache works again (thanks @jchv). (cherry picked from commit 9b57573baea5abd242c5f62f537c7582c0097c3b) --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb70fae87..29cb33f56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: extra_nix_config: | sandbox = true max-jobs = 1 + - uses: DeterminateSystems/magic-nix-cache-action@main # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 @@ -133,6 +134,7 @@ jobs: - uses: cachix/install-nix-action@v31 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install + - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz @@ -174,6 +176,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main - run: | nix build -L \ .#hydraJobs.tests.functional_user \ @@ -199,4 +202,5 @@ jobs: repository: NixOS/flake-regressions-data path: flake-regressions/tests - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh From 5d2986d3c52b49ea82f9e2ea4d9a86929b55121a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= Date: Wed, 18 Jun 2025 10:05:02 +0200 Subject: [PATCH 22/31] tests: fixup with jq-1.8.0 --- tests/functional/flakes/flakes.sh | 4 ++-- tests/functional/flakes/relative-paths.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 261d65d69..a433cf71f 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -163,11 +163,11 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] -[[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] if [[ $(nix config show lazy-trees) = true ]]; then # Test that `lazy-locks` causes NAR hashes to be omitted from the lock file. nix flake update --flake "$flake2Dir" --commit-lock-file --lazy-locks - [[ $(jq --indent 0 . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] + [[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] fi # Rerunning the build should not change the lockfile. diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 9d31da0ad..7a76bee1b 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -69,7 +69,7 @@ git -C "$rootFlake" add flake.nix sub2/flake.nix git -C "$rootFlake" add sub2/flake.lock [[ $(nix eval "$subflake2#y") = 15 ]] -[[ $(jq --indent 0 . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" "$subflake2/flake.lock") From 833406121cb65c42de8e0c4fad62be140b1b7978 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Jun 2025 19:14:31 +0200 Subject: [PATCH 23/31] Make the repl test more robust Seen in https://github.com/DeterminateSystems/nix-src/actions/runs/15590867877/job/43909540271: nix-functional-tests> grep: repl_output: No such file or directory nix-functional-tests> +(repl.sh:174) cat repl_output This is because there is a small possibility that the `nix repl` child process hasn't created `repl_output` yet. So make sure it exists. (cherry picked from commit 9eb46e9cc030016b1f4a073474a836bac1de3615) --- tests/functional/repl.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 762636e44..82a932e2b 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -163,7 +163,8 @@ foo + baz # - Re-eval it # - Check that the result has changed mkfifo repl_fifo -nix repl ./flake --experimental-features 'flakes' < repl_fifo > repl_output 2>&1 & +touch repl_output +nix repl ./flake --experimental-features 'flakes' < repl_fifo >> repl_output 2>&1 & repl_pid=$! exec 3>repl_fifo # Open fifo for writing echo "changingThing" >&3 From 58a878d846ffadd016595e9ed9d07fceac2199a1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 13:18:05 +0000 Subject: [PATCH 24/31] Prepare release v3.6.7 From ff05659f8bc5c84568e0ead4dbee1a8eb9705ee2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 13:18:08 +0000 Subject: [PATCH 25/31] Set .version-determinate to 3.6.7 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 4f2c1d15f..5b3413147 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.6 +3.6.7 From 098be10e285b05e6b04d3d7feb14270a9daefba1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 13:18:13 +0000 Subject: [PATCH 26/31] Generate release notes for 3.6.7 --- doc/manual/source/SUMMARY.md.in | 1 + .../source/release-notes-determinate/changes.md | 10 +++++++++- .../source/release-notes-determinate/rl-3.6.7.md | 11 +++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.7.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index dd3218d2f..b4458fc8c 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.7 (2025-06-24)](release-notes-determinate/rl-3.6.7.md) - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md) - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 6f27f7f6b..f3183883c 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.6. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.7. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -74,3 +74,11 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + + + +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) + +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) + +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.7.md b/doc/manual/source/release-notes-determinate/rl-3.6.7.md new file mode 100644 index 000000000..6ce425217 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.7.md @@ -0,0 +1,11 @@ +# Release 3.6.7 (2025-06-24) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) + + +**Full Changelog**: [v3.6.6...v3.6.7](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.7) From 731b63032161d3712ae26825230048dc5875eef4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Jun 2025 16:20:34 +0200 Subject: [PATCH 27/31] Fixes for GHSA-g948-229j-48j3 Squashed commit of the following: commit 04fff3a637d455cbb1d75937a235950e43008db9 Author: Eelco Dolstra Date: Thu Jun 12 12:30:32 2025 +0200 Chown structured attr files safely commit 5417ad445e414c649d0cfc71a05661c7bf8f3ef5 Author: Eelco Dolstra Date: Thu Jun 12 12:14:04 2025 +0200 Replace 'bool sync' with an enum for clarity And drop writeFileAndSync(). commit 7ae0141f328d8e8e1094be24665789c05f974ba6 Author: Eelco Dolstra Date: Thu Jun 12 11:35:28 2025 +0200 Drop guessOrInventPathFromFD() No need to do hacky stuff like that when we already know the original path. commit 45b05098bd019da7c57cd4227a89bfd0fa65bb08 Author: Eelco Dolstra Date: Thu Jun 12 11:15:58 2025 +0200 Tweak comment commit 0af15b31209d1b7ec8addfae9a1a6b60d8f35848 Author: Raito Bezarius Date: Thu Mar 27 12:22:26 2025 +0100 libstore: ensure that temporary directory is always 0o000 before deletion In the case the deletion fails, we should ensure that the temporary directory cannot be used for nefarious purposes. Change-Id: I498a2dd0999a74195d13642f44a5de1e69d46120 Signed-off-by: Raito Bezarius commit 2c20fa37b15cfa03ac6a1a6a47cdb2ed66c0827e Author: Raito Bezarius Date: Wed Mar 26 12:42:55 2025 +0100 libutil: ensure that `_deletePath` does NOT use absolute paths with dirfds When calling `_deletePath` with a parent file descriptor, `openat` is made effective by using relative paths to the directory file descriptor. To avoid the problem, the signature is changed to resist misuse with an assert in the prologue of the function. Change-Id: I6b3fc766bad2afe54dc27d47d1df3873e188de96 Signed-off-by: Raito Bezarius commit d3c370bbcae48bb825ce19fd0f73bb4eefd2c9ea Author: Raito Bezarius Date: Wed Mar 26 01:07:47 2025 +0100 libstore: ensure that `passAsFile` is created in the original temp dir This ensures that `passAsFile` data is created inside the expected temporary build directory by `openat()` from the parent directory file descriptor. This avoids a TOCTOU which is part of the attack chain of CVE-????. Change-Id: Ie5273446c4a19403088d0389ae8e3f473af8879a Signed-off-by: Raito Bezarius commit 45d3598724f932d024ef6bc2ffb00c1bb90e6018 Author: Raito Bezarius Date: Wed Mar 26 01:06:03 2025 +0100 libutil: writeFile variant for file descriptors `writeFile` lose its `sync` boolean flag to make things simpler. A new `writeFileAndSync` function is created and all call sites are converted to it. Change-Id: Ib871a5283a9c047db1e4fe48a241506e4aab9192 Signed-off-by: Raito Bezarius commit 732bd9b98cabf4aaf95a01fd318923de303f9996 Author: Raito Bezarius Date: Wed Mar 26 01:05:34 2025 +0100 libstore: chown to builder variant for file descriptors We use it immediately for the build temporary directory. Change-Id: I180193c63a2b98721f5fb8e542c4e39c099bb947 Signed-off-by: Raito Bezarius commit 962c65f8dcd5570dd92c72370a862c7b38942e0d Author: Raito Bezarius Date: Wed Mar 26 01:04:59 2025 +0100 libstore: open build directory as a dirfd as well We now keep around a proper AutoCloseFD around the temporary directory which we plan to use for openat operations and avoiding the build directory being swapped out while we are doing something else. Change-Id: I18d387b0f123ebf2d20c6405cd47ebadc5505f2a Signed-off-by: Raito Bezarius commit c9b42462b75b5a37ee6564c2b53cff186c8323da Author: Raito Bezarius Date: Wed Mar 26 01:04:12 2025 +0100 libutil: guess or invent a path from file descriptors This is useful for certain error recovery paths (no pun intended) that does not thread through the original path name. Change-Id: I2d800740cb4f9912e64c923120d3f977c58ccb7e Signed-off-by: Raito Bezarius --- src/libstore/local-store.cc | 4 +- src/libstore/unix/build/derivation-builder.cc | 66 ++++++++++++++++--- src/libutil/file-content-address.cc | 2 +- src/libutil/file-system.cc | 53 ++++++++------- src/libutil/include/nix/util/file-system.hh | 14 ++-- 5 files changed, 101 insertions(+), 38 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 76fadba86..1ab3ed13a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -247,7 +247,7 @@ LocalStore::LocalStore(ref config) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(*state, true); - writeFile(schemaPath, fmt("%1%", curSchema), 0666, true); + writeFile(schemaPath, fmt("%1%", curSchema), 0666, FsSync::Yes); } else if (curSchema < nixSchemaVersion) { @@ -298,7 +298,7 @@ LocalStore::LocalStore(ref config) txn.commit(); } - writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, FsSync::Yes); lockFile(globalLock.get(), ltRead, true); } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index e84e2db6e..43dfe1832 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -129,6 +129,11 @@ private: */ Path topTmpDir; + /** + * The file descriptor of the temporary directory. + */ + AutoCloseFD tmpDirFd; + /** * The path of the temporary directory in the sandbox. */ @@ -325,9 +330,24 @@ private: /** * Make a file owned by the builder. + * + * SAFETY: this function is prone to TOCTOU as it receives a path and not a descriptor. + * It's only safe to call in a child of a directory only visible to the owner. */ void chownToBuilder(const Path & path); + /** + * Make a file owned by the builder addressed by its file descriptor. + */ + void chownToBuilder(int fd, const Path & path); + + /** + * Create a file in `tmpDir` owned by the builder. + */ + void writeBuilderFile( + const std::string & name, + std::string_view contents); + /** * Run the builder's process. */ @@ -900,7 +920,14 @@ void DerivationBuilderImpl::startBuilder() } else { tmpDir = topTmpDir; } - chownToBuilder(tmpDir); + + /* The TOCTOU between the previous mkdir call and this open call is unavoidable due to + POSIX semantics.*/ + tmpDirFd = AutoCloseFD{open(tmpDir.c_str(), O_RDONLY | O_NOFOLLOW | O_DIRECTORY)}; + if (!tmpDirFd) + throw SysError("failed to open the build temporary directory descriptor '%1%'", tmpDir); + + chownToBuilder(tmpDirFd.get(), tmpDir); for (auto & [outputName, status] : initialOutputs) { /* Set scratch path we'll actually use during the build. @@ -1485,9 +1512,7 @@ void DerivationBuilderImpl::initTmpDir() } else { auto hash = hashString(HashAlgorithm::SHA256, i.first); std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); + writeBuilderFile(fn, rewriteStrings(i.second, inputRewrites)); env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } } @@ -1596,11 +1621,9 @@ void DerivationBuilderImpl::writeStructuredAttrs() auto jsonSh = StructuredAttrs::writeShell(json); - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); + writeBuilderFile(".attrs.sh", rewriteStrings(jsonSh, inputRewrites)); env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); + writeBuilderFile(".attrs.json", rewriteStrings(json.dump(), inputRewrites)); env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; } } @@ -1854,6 +1877,24 @@ void setupSeccomp() #endif } +void DerivationBuilderImpl::chownToBuilder(int fd, const Path & path) +{ + if (!buildUser) return; + if (fchown(fd, buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError("cannot change ownership of file '%1%'", path); +} + +void DerivationBuilderImpl::writeBuilderFile( + const std::string & name, + std::string_view contents) +{ + auto path = std::filesystem::path(tmpDir) / name; + AutoCloseFD fd{openat(tmpDirFd.get(), name.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666)}; + if (!fd) + throw SysError("creating file %s", path); + writeFile(fd, path, contents); + chownToBuilder(fd.get(), path); +} void DerivationBuilderImpl::runChild() { @@ -3065,6 +3106,15 @@ void DerivationBuilderImpl::checkOutputs(const std::mapd_name; if (childName == "." || childName == "..") continue; - _deletePath(dirfd(dir.get()), path + "/" + childName, bytesFreed, ex); + _deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex); } if (errno) throw SysError("reading directory %1%", path); } @@ -497,14 +505,13 @@ static void _deletePath(Descriptor parentfd, const std::filesystem::path & path, static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed) { - Path dir = dirOf(path.string()); - if (dir == "") - dir = "/"; + assert(path.is_absolute()); + assert(path.parent_path() != path); - AutoCloseFD dirfd = toDescriptor(open(dir.c_str(), O_RDONLY)); + AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY)); if (!dirfd) { if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); + throw SysError("opening directory %s", path.parent_path()); } std::exception_ptr ex; diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index b8fa4cfa0..a9a6e43bf 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -175,21 +175,27 @@ std::string readFile(const Path & path); std::string readFile(const std::filesystem::path & path); void readFile(const Path & path, Sink & sink, bool memory_map = true); +enum struct FsSync { Yes, No }; + /** * Write a string to a file. */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); -static inline void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, bool sync = false) +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No); + +static inline void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No) { return writeFile(path.string(), s, mode, sync); } -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); -static inline void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, bool sync = false) +void writeFile(const Path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No); + +static inline void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No) { return writeFile(path.string(), source, mode, sync); } +void writeFile(AutoCloseFD & fd, const Path & origPath, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No); + /** * Flush a path's parent directory to disk. */ From 404d82419405c31dd27a2fb27b387f0497fb26c4 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 24 Jun 2025 09:29:07 -0400 Subject: [PATCH 28/31] Update changelogs --- .../source/release-notes-determinate/changes.md | 6 ------ .../source/release-notes-determinate/rl-3.6.7.md | 12 +++++++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index f3183883c..95374dcb6 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -76,9 +76,3 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix - -* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) - -* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) - -* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.7.md b/doc/manual/source/release-notes-determinate/rl-3.6.7.md index 6ce425217..197587f1b 100644 --- a/doc/manual/source/release-notes-determinate/rl-3.6.7.md +++ b/doc/manual/source/release-notes-determinate/rl-3.6.7.md @@ -3,9 +3,15 @@ * Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). ## What's Changed -* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) -* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) -* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) + +### Security contents + +* Patched against GHSA-g948-229j-48j3 + +### Lazy trees: + +* Lazy trees now produces `flake.lock` files with NAR hashes unless `lazy-locks` is set to `true` by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Improved caching with lazy-trees when using --impure, with enhanced testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) **Full Changelog**: [v3.6.6...v3.6.7](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.7) From c2baff64f7f9d0c6a5135acfdca483c9c9f41712 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 03:32:32 +0000 Subject: [PATCH 29/31] Prepare release v3.6.8 From fd8b3270602406aa492af4d32ee47ccc3a9d6f03 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 03:32:35 +0000 Subject: [PATCH 30/31] Set .version-determinate to 3.6.8 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 5b3413147..424e1794d 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.6.7 +3.6.8 From 4ba66db9f66297412be45385d6ed1654fda0408f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 03:32:40 +0000 Subject: [PATCH 31/31] Generate release notes for 3.6.8 --- doc/manual/source/SUMMARY.md.in | 1 + .../source/release-notes-determinate/changes.md | 12 +++++++++++- .../source/release-notes-determinate/rl-3.6.8.md | 12 ++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/rl-3.6.8.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index b4458fc8c..ebcb7b956 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -129,6 +129,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.6.8 (2025-06-25)](release-notes-determinate/rl-3.6.8.md) - [Release 3.6.7 (2025-06-24)](release-notes-determinate/rl-3.6.7.md) - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md) - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 95374dcb6..3a38378e0 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.7. +This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.8. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -76,3 +76,13 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix + + + +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) + +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) + +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) + +* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.8.md b/doc/manual/source/release-notes-determinate/rl-3.6.8.md new file mode 100644 index 000000000..c4b4b96c9 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.8.md @@ -0,0 +1,12 @@ +# Release 3.6.8 (2025-06-25) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) +* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126) + + +**Full Changelog**: [v3.6.6...v3.6.8](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.8)