From 0bfa0cdea1b4eb09405e35b338887c91a041d28c Mon Sep 17 00:00:00 2001 From: Martin Schwaighofer Date: Mon, 13 Dec 2021 21:31:15 +0100 Subject: [PATCH 01/41] git fetcher: improve check for valid repository The .git/refs/heads directory might be empty for a valid usable git repository. This often happens in CI environments, which might only fetch commits, not branches. Therefore instead we let git itself check if HEAD points to something that looks like a commit. fixes #5302 --- src/libfetchers/git.cc | 28 ++++++++++++++-------------- tests/fetchGit.sh | 8 ++++++++ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c3f0f8c8f..7479318e3 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -220,21 +220,21 @@ struct GitInputScheme : InputScheme if (!input.getRef() && !input.getRev() && isLocal) { bool clean = false; - /* Check whether this repo has any commits. There are - probably better ways to do this. */ - auto gitDir = actualUrl + "/.git"; - auto commonGitDir = chomp(runProgram( - "git", - true, - { "-C", actualUrl, "rev-parse", "--git-common-dir" } - )); - if (commonGitDir != ".git") - gitDir = commonGitDir; - - bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty(); + /* Check whether HEAD points to something that looks like a commit, + since that is the refrence we want to use later on. */ + bool hasHead = false; + try { + runProgram("git", true, { "-C", actualUrl, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }); + hasHead = true; + } catch (ExecError & e) { + // git exits with status 128 here if it does not detect a repository. + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 128) { + throw Error("Git tree '%s' is broken.\n'git rev-parse --verify HEAD' failed with exit code %d.", actualUrl, WEXITSTATUS(e.status)); + } + } try { - if (haveCommits) { + if (hasHead) { runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); clean = true; } @@ -280,7 +280,7 @@ struct GitInputScheme : InputScheme // modified dirty file? input.attrs.insert_or_assign( "lastModified", - haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + hasHead ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return {std::move(storePath), input}; } diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 89294d8d2..dd0d98956 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -170,6 +170,14 @@ NIX=$(command -v nix) path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] +# Fetching from a repo with only a specific revision and no branches should +# not fall back to copying files and record correct revision information. See: #5302 +mkdir $TEST_ROOT/minimal +git -C $TEST_ROOT/minimal init +git -C $TEST_ROOT/minimal fetch $repo $rev2 +git -C $TEST_ROOT/minimal checkout $rev2 +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit { url = $TEST_ROOT/minimal; }).rev") = $rev2 ]] + # Fetching a shallow repo shouldn't work by default, because we can't # return a revCount. git clone --depth 1 file://$repo $TEST_ROOT/shallow From c7e527b82b3dafed5f0da2ec0e14a47cf8e65def Mon Sep 17 00:00:00 2001 From: Martin Schwaighofer Date: Mon, 13 Dec 2021 21:31:20 +0100 Subject: [PATCH 02/41] git fetcher: invoke diff instead of diff-index diff-index operates on the view that git has of the working tree, which might be outdated. The higher-level diff command does this automatically. This change also adds handling for submodules. fixes #4140 Alternative fixes would be invoking update-index before diff-index or matching more closely what require_clean_work_tree from git-sh-setup.sh does, but both those options make it more difficult to reason about correctness. --- src/libfetchers/git.cc | 12 +++++++++++- tests/fetchGit.sh | 7 ++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7479318e3..d241eb67a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -235,7 +235,17 @@ struct GitInputScheme : InputScheme try { if (hasHead) { - runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); + // Using git diff is preferrable over lower-level operations here, + // because its conceptually simpler and we only need the exit code anyways. + auto gitDiffOpts = Strings({ "-C", actualUrl, "diff", "HEAD", "--quiet"}); + if (!submodules) { + // Changes in submodules should only make the tree dirty + // when those submodules will be copied as well. + gitDiffOpts.emplace_back("--ignore-submodules"); + } + gitDiffOpts.emplace_back("--"); + runProgram("git", true, gitDiffOpts); + clean = true; } } catch (ExecError & e) { diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index dd0d98956..628d96924 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -11,7 +11,7 @@ repo=$TEST_ROOT/git export _NIX_FORCE_HTTP=1 -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow $TEST_ROOT/minimal git init $repo git -C $repo config user.email "foobar@example.com" @@ -147,8 +147,13 @@ path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).shortRev") = 0000000 ]] +# Making a dirty tree clean again and fetching it should +# record correct revision information. See: #4140 +echo world > $repo/hello +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = $rev2 ]] # Committing shouldn't change store path, or switch to using 'master' +echo dev > $repo/hello git -C $repo commit -m 'Bla5' -a path4=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $(cat $path4/hello) = dev ]] From 9504445cab095fe3869c5a68342fb2cf23ac0f28 Mon Sep 17 00:00:00 2001 From: Martin Schwaighofer Date: Sat, 1 Jan 2022 21:26:41 +0100 Subject: [PATCH 03/41] git fetcher: distinguish errors more precisely --- src/libfetchers/git.cc | 26 +++++++++++++++++--------- tests/fetchGit.sh | 8 ++++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d241eb67a..ad877eacc 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -222,17 +222,25 @@ struct GitInputScheme : InputScheme /* Check whether HEAD points to something that looks like a commit, since that is the refrence we want to use later on. */ - bool hasHead = false; - try { - runProgram("git", true, { "-C", actualUrl, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }); - hasHead = true; - } catch (ExecError & e) { - // git exits with status 128 here if it does not detect a repository. - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 128) { - throw Error("Git tree '%s' is broken.\n'git rev-parse --verify HEAD' failed with exit code %d.", actualUrl, WEXITSTATUS(e.status)); - } + auto result = runProgram(RunOptions { + .program = "git", + .args = { "-C", actualUrl, "--git-dir=.git", "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, + .mergeStderrToStdout = true + }); + auto exitCode = WEXITSTATUS(result.first); + auto errorMessage = result.second; + + if (errorMessage.find("fatal: not a git repository") != std::string::npos) { + throw Error("'%s' is not a git repository.", actualUrl); + } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { + // indicates that the repo does not have any commits + // we want to proceed and will consider it dirty later + } else if (exitCode != 0) { + // any other errors should lead to a failure + throw Error("Getting the HEAD of the git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage); } + bool hasHead = exitCode == 0; try { if (hasHead) { // Using git diff is preferrable over lower-level operations here, diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 628d96924..ac23be5c0 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -206,3 +206,11 @@ rev4_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$ # The name argument should be handled path9=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; name = \"foo\"; }).outPath") [[ $path9 =~ -foo$ ]] + +# should fail if there is no repo +rm -rf $repo/.git +(! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") + +# should succeed for a repo without commits +git init $repo +path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") From 53523c0ab834416e38a15cf7be6f71d8f68d1c99 Mon Sep 17 00:00:00 2001 From: Martin Schwaighofer Date: Mon, 7 Feb 2022 20:36:39 +0100 Subject: [PATCH 04/41] git fetcher: set locale for rev-parse --- src/libfetchers/git.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ad877eacc..6571a9d02 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -220,11 +220,17 @@ struct GitInputScheme : InputScheme if (!input.getRef() && !input.getRev() && isLocal) { bool clean = false; + auto env = getEnv(); + // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong + // that way unknown errors can lead to a failure instead of continuing through the wrong code path + env["LC_ALL"] = "C"; + /* Check whether HEAD points to something that looks like a commit, since that is the refrence we want to use later on. */ auto result = runProgram(RunOptions { .program = "git", .args = { "-C", actualUrl, "--git-dir=.git", "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, + .environment = env, .mergeStderrToStdout = true }); auto exitCode = WEXITSTATUS(result.first); From 0c6e46e34965ba0db4e0b755ee473868a4fef21f Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Mar 2022 06:16:51 +0100 Subject: [PATCH 05/41] Add some suggestions to the evaluator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the evaluator show some suggestions when trying to access an invalid field from an attrset. ```console $ nix eval --expr '{ foo = 1; }.foa' error: attribute 'foa' missing at «string»:1:1: 1| { foo = 1; }.foa | ^ Did you mean foo? ``` --- src/libexpr/eval.cc | 20 ++++++++++++++++++-- tests/suggestions.sh | 4 ++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5bf161cc0..1d88e8709 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -727,6 +727,15 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const std::string & s2 throw EvalError(s, s2); } +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const Suggestions & suggestions, const char * s, const std::string & s2)) +{ + throw EvalError({ + .msg = hintfmt(s, s2), + .errPos = pos, + .suggestions = suggestions, + }); +} + LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2)) { throw EvalError({ @@ -1281,8 +1290,15 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } } else { state.forceAttrs(*vAttrs, pos); - if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - throwEvalError(pos, "attribute '%1%' missing", name); + if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + std::set allAttrNames; + for (auto & attr : *vAttrs->attrs) + allAttrNames.insert(attr.name); + throwEvalError( + pos, + Suggestions::bestMatches(allAttrNames, name), + "attribute '%1%' missing", name); + } } vAttrs = j->value; pos2 = j->pos; diff --git a/tests/suggestions.sh b/tests/suggestions.sh index 16a5a7004..29d5b364b 100644 --- a/tests/suggestions.sh +++ b/tests/suggestions.sh @@ -34,3 +34,7 @@ NIX_BUILD_STDERR_WITH_SUGGESTIONS=$(! nix build .\#fob 2>&1 1>/dev/null) NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) [[ ! "$NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION" =~ "Did you mean" ]] || \ fail "The nix build stderr shouldn’t suggest anything if there’s nothing relevant to suggest" + +NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'$system'.fob' 2>&1 1>/dev/null) +[[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ + fail "The evaluator should suggest the three closest possiblities" From 33b7514035a967df2ab61ab9770627157aa4f5c5 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Mar 2022 16:07:17 +0100 Subject: [PATCH 06/41] Try and make the darwin build happy --- src/libexpr/eval.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1d88e8709..3bfb82b16 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -729,7 +729,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const std::string & s2 LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const Suggestions & suggestions, const char * s, const std::string & s2)) { - throw EvalError({ + throw EvalError(ErrorInfo { .msg = hintfmt(s, s2), .errPos = pos, .suggestions = suggestions, @@ -738,7 +738,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const Suggestions & s LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2)) { - throw EvalError({ + throw EvalError(ErrorInfo { .msg = hintfmt(s, s2), .errPos = pos }); From f6078e474d5fc41c8a7f683865d60490bf0c7040 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Mar 2022 16:20:01 +0100 Subject: [PATCH 07/41] Also display some suggestions for invalid formal arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ```console $ nix eval --expr '({ foo ? 1 }: foo) { fob = 2; }' error: anonymous function at (string):1:2 called with unexpected argument 'fob' at «string»:1:1: 1| ({ foo ? 1 }: foo) { fob = 2; } | ^ Did you mean foo? ``` Not that because Nix will first check for _missing_ arguments before checking for extra arguments, `({ foo }: foo) { fob = 1; }` will complain about the missing `foo` argument (rather than extra `fob`) and so won’t display a suggestion. --- src/libexpr/eval.cc | 23 +++++++++++++++++++++-- tests/suggestions.sh | 4 ++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3bfb82b16..a5e9dc286 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -782,6 +782,16 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const }); } +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const Suggestions & suggestions, const char * s, const ExprLambda & fun, const Symbol & s2)) +{ + throw TypeError(ErrorInfo { + .msg = hintfmt(s, fun.showNamePos(), s2), + .errPos = pos, + .suggestions = suggestions, + }); +} + + LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) { throw TypeError(s, showType(v)); @@ -1414,8 +1424,17 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* Nope, so show the first unexpected argument to the user. */ for (auto & i : *args[0]->attrs) - if (!lambda.formals->has(i.name)) - throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); + if (!lambda.formals->has(i.name)) { + std::set formalNames; + for (auto & formal : lambda.formals->formals) + formalNames.insert(formal.name); + throwTypeError( + pos, + Suggestions::bestMatches(formalNames, i.name), + "%1% called with unexpected argument '%2%'", + lambda, + i.name); + } abort(); // can't happen } } diff --git a/tests/suggestions.sh b/tests/suggestions.sh index 29d5b364b..f18fefef9 100644 --- a/tests/suggestions.sh +++ b/tests/suggestions.sh @@ -38,3 +38,7 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'$system'.fob' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ fail "The evaluator should suggest the three closest possiblities" + +NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '({ foo }: foo) { foo = 1; fob = 2; }' 2>&1 1>/dev/null) +[[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean foo?" ]] || \ + fail "The evaluator should suggest the three closest possiblities" From 4b2b0d3a5528133898a15f3210c79162ec993823 Mon Sep 17 00:00:00 2001 From: pennae Date: Wed, 29 Dec 2021 01:28:58 +0100 Subject: [PATCH 08/41] remove GC_PTR_STORE_AND_DIRTY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit turns out it's only necessary for MANUAL_VDB, which nix doesn't use. omitting them gives a slight performance improvement on eval. before: Benchmark 1: nix flakes search --no-eval-cache --offline ../nixpkgs hello Time (mean ± σ): 6.988 s ± 0.061 s [User: 5.935 s, System: 0.845 s] Range (min … max): 6.865 s … 7.075 s 20 runs Benchmark 2: nix flakes eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix Time (mean ± σ): 332.6 ms ± 3.9 ms [User: 299.6 ms, System: 32.9 ms] Range (min … max): 328.1 ms … 339.1 ms 20 runs Benchmark 3: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 2.681 s ± 0.049 s [User: 2.382 s, System: 0.228 s] Range (min … max): 2.607 s … 2.776 s 20 runs after: Benchmark 1: nix flakes search --no-eval-cache --offline ../nixpkgs hello Time (mean ± σ): 6.946 s ± 0.041 s [User: 5.875 s, System: 0.835 s] Range (min … max): 6.834 s … 7.005 s 20 runs Benchmark 2: nix flakes eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix Time (mean ± σ): 330.3 ms ± 2.5 ms [User: 299.2 ms, System: 30.9 ms] Range (min … max): 327.5 ms … 337.7 ms 20 runs Benchmark 3: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 2.671 s ± 0.035 s [User: 2.370 s, System: 0.232 s] Range (min … max): 2.597 s … 2.749 s 20 runs --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 193358161..777f6a4ec 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -879,7 +879,7 @@ Value * EvalState::allocValue() /* GC_NEXT is a convenience macro for accessing the first word of an object. Take the first list item, advance the list to the next item, and clear the next pointer. */ void * p = *valueAllocCache; - GC_PTR_STORE_AND_DIRTY(&*valueAllocCache, GC_NEXT(p)); + *valueAllocCache = GC_NEXT(p); GC_NEXT(p) = nullptr; nrValues++; From 60ed4e908a59f258f82356a9dd47e43361d39f2f Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 26 Dec 2021 19:32:08 +0100 Subject: [PATCH 09/41] cache singleton Envs just like Values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vast majority of envs is this size. before: Benchmark 1: nix flakes search --no-eval-cache --offline ../nixpkgs hello Time (mean ± σ): 6.946 s ± 0.041 s [User: 5.875 s, System: 0.835 s] Range (min … max): 6.834 s … 7.005 s 20 runs Benchmark 2: nix flakes eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix Time (mean ± σ): 330.3 ms ± 2.5 ms [User: 299.2 ms, System: 30.9 ms] Range (min … max): 327.5 ms … 337.7 ms 20 runs Benchmark 3: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 2.671 s ± 0.035 s [User: 2.370 s, System: 0.232 s] Range (min … max): 2.597 s … 2.749 s 20 runs after: Benchmark 1: nix flakes search --no-eval-cache --offline ../nixpkgs hello Time (mean ± σ): 6.935 s ± 0.052 s [User: 5.852 s, System: 0.853 s] Range (min … max): 6.808 s … 7.026 s 20 runs Benchmark 2: nix flakes eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix Time (mean ± σ): 329.8 ms ± 2.7 ms [User: 299.0 ms, System: 30.8 ms] Range (min … max): 326.6 ms … 336.5 ms 20 runs Benchmark 3: nix flakes eval --raw --impure --file expr.nix Time (mean ± σ): 2.655 s ± 0.038 s [User: 2.364 s, System: 0.220 s] Range (min … max): 2.574 s … 2.737 s 20 runs --- src/libexpr/eval.cc | 21 ++++++++++++++++++++- src/libexpr/eval.hh | 3 +++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 777f6a4ec..ddbbdeaa6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -438,8 +438,10 @@ EvalState::EvalState( , regexCache(makeRegexCache()) #if HAVE_BOEHMGC , valueAllocCache(std::allocate_shared(traceable_allocator(), nullptr)) + , env1AllocCache(std::allocate_shared(traceable_allocator(), nullptr)) #else , valueAllocCache(std::make_shared(nullptr)) + , env1AllocCache(std::make_shared(nullptr)) #endif , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) @@ -892,7 +894,24 @@ Env & EvalState::allocEnv(size_t size) { nrEnvs++; nrValuesInEnvs += size; - Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); + + Env * env; + + if (size != 1) + env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); + else { + /* see allocValue for explanations. */ + if (!*env1AllocCache) { + *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *)); + if (!*env1AllocCache) throw std::bad_alloc(); + } + + void * p = *env1AllocCache; + *env1AllocCache = GC_NEXT(p); + GC_NEXT(p) = nullptr; + env = (Env *) p; + } + env->type = Env::Plain; /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 800b00eef..41384f044 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -136,6 +136,9 @@ private: /* Allocation cache for GC'd Value objects. */ std::shared_ptr valueAllocCache; + /* Allocation cache for size-1 Env objects. */ + std::shared_ptr env1AllocCache; + public: EvalState( From c96460f3520862d52b7bf3108f609e20384878e7 Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 28 Dec 2021 19:18:17 +0100 Subject: [PATCH 10/41] force-inline a few much-used functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit these functions are called a whole lot, and they're all comparatively small. always inlining them gives ~0.7% performance boost on eval. before: Benchmark 1: nix flakes search --no-eval-cache --offline ../nixpkgs hello Time (mean ± σ): 6.935 s ± 0.052 s [User: 5.852 s, System: 0.853 s] Range (min … max): 6.808 s … 7.026 s 20 runs Benchmark 2: nix flakes eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix Time (mean ± σ): 329.8 ms ± 2.7 ms [User: 299.0 ms, System: 30.8 ms] Range (min … max): 326.6 ms … 336.5 ms 20 runs Benchmark 3: nix flakes eval --raw --impure --file expr.nix Time (mean ± σ): 2.655 s ± 0.038 s [User: 2.364 s, System: 0.220 s] Range (min … max): 2.574 s … 2.737 s 20 runs after: Benchmark 1: nix flakes search --no-eval-cache --offline ../nixpkgs hello Time (mean ± σ): 6.912 s ± 0.036 s [User: 5.823 s, System: 0.856 s] Range (min … max): 6.849 s … 6.980 s 20 runs Benchmark 2: nix flakes eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix Time (mean ± σ): 325.1 ms ± 2.5 ms [User: 293.2 ms, System: 31.8 ms] Range (min … max): 322.2 ms … 332.8 ms 20 runs Benchmark 3: nix flakes eval --raw --impure --file expr.nix Time (mean ± σ): 2.636 s ± 0.024 s [User: 2.352 s, System: 0.226 s] Range (min … max): 2.574 s … 2.681 s 20 runs --- src/libexpr/eval-inline.hh | 86 ++++++++++++++++++++++++++++++++------ src/libexpr/eval.cc | 53 ----------------------- src/libexpr/eval.hh | 6 ++- 3 files changed, 77 insertions(+), 68 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index aef1f6351..3331a7643 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -24,6 +24,76 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const } +/* Note: Various places expect the allocated memory to be zeroed. */ +[[gnu::always_inline]] +inline void * allocBytes(size_t n) +{ + void * p; +#if HAVE_BOEHMGC + p = GC_MALLOC(n); +#else + p = calloc(n, 1); +#endif + if (!p) throw std::bad_alloc(); + return p; +} + + +[[gnu::always_inline]] +Value * EvalState::allocValue() +{ + /* We use the boehm batch allocator to speed up allocations of Values (of which there are many). + GC_malloc_many returns a linked list of objects of the given size, where the first word + of each object is also the pointer to the next object in the list. This also means that we + have to explicitly clear the first word of every object we take. */ + if (!*valueAllocCache) { + *valueAllocCache = GC_malloc_many(sizeof(Value)); + if (!*valueAllocCache) throw std::bad_alloc(); + } + + /* GC_NEXT is a convenience macro for accessing the first word of an object. + Take the first list item, advance the list to the next item, and clear the next pointer. */ + void * p = *valueAllocCache; + *valueAllocCache = GC_NEXT(p); + GC_NEXT(p) = nullptr; + + nrValues++; + return (Value *) p; +} + + +[[gnu::always_inline]] +Env & EvalState::allocEnv(size_t size) +{ + nrEnvs++; + nrValuesInEnvs += size; + + Env * env; + + if (size != 1) + env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); + else { + /* see allocValue for explanations. */ + if (!*env1AllocCache) { + *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *)); + if (!*env1AllocCache) throw std::bad_alloc(); + } + + void * p = *env1AllocCache; + *env1AllocCache = GC_NEXT(p); + GC_NEXT(p) = nullptr; + env = (Env *) p; + } + + env->type = Env::Plain; + + /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ + + return *env; +} + + +[[gnu::always_inline]] void EvalState::forceValue(Value & v, const Pos & pos) { forceValue(v, [&]() { return pos; }); @@ -52,6 +122,7 @@ void EvalState::forceValue(Value & v, Callable getPos) } +[[gnu::always_inline]] inline void EvalState::forceAttrs(Value & v, const Pos & pos) { forceAttrs(v, [&]() { return pos; }); @@ -59,6 +130,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos) template +[[gnu::always_inline]] inline void EvalState::forceAttrs(Value & v, Callable getPos) { forceValue(v, getPos); @@ -67,6 +139,7 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos) } +[[gnu::always_inline]] inline void EvalState::forceList(Value & v, const Pos & pos) { forceValue(v, pos); @@ -74,18 +147,5 @@ inline void EvalState::forceList(Value & v, const Pos & pos) throwTypeError(pos, "value is %1% while a list was expected", v); } -/* Note: Various places expect the allocated memory to be zeroed. */ -inline void * allocBytes(size_t n) -{ - void * p; -#if HAVE_BOEHMGC - p = GC_MALLOC(n); -#else - p = calloc(n, 1); -#endif - if (!p) throw std::bad_alloc(); - return p; -} - } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index ddbbdeaa6..038b6bb7c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -867,59 +867,6 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) } -Value * EvalState::allocValue() -{ - /* We use the boehm batch allocator to speed up allocations of Values (of which there are many). - GC_malloc_many returns a linked list of objects of the given size, where the first word - of each object is also the pointer to the next object in the list. This also means that we - have to explicitly clear the first word of every object we take. */ - if (!*valueAllocCache) { - *valueAllocCache = GC_malloc_many(sizeof(Value)); - if (!*valueAllocCache) throw std::bad_alloc(); - } - - /* GC_NEXT is a convenience macro for accessing the first word of an object. - Take the first list item, advance the list to the next item, and clear the next pointer. */ - void * p = *valueAllocCache; - *valueAllocCache = GC_NEXT(p); - GC_NEXT(p) = nullptr; - - nrValues++; - auto v = (Value *) p; - return v; -} - - -Env & EvalState::allocEnv(size_t size) -{ - nrEnvs++; - nrValuesInEnvs += size; - - Env * env; - - if (size != 1) - env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); - else { - /* see allocValue for explanations. */ - if (!*env1AllocCache) { - *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *)); - if (!*env1AllocCache) throw std::bad_alloc(); - } - - void * p = *env1AllocCache; - *env1AllocCache = GC_NEXT(p); - GC_NEXT(p) = nullptr; - env = (Env *) p; - } - - env->type = Env::Plain; - - /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ - - return *env; -} - - void EvalState::mkList(Value & v, size_t size) { v.mkList(size); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 41384f044..d2efe8a47 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -350,8 +350,8 @@ public: void autoCallFunction(Bindings & args, Value & fun, Value & res); /* Allocation primitives. */ - Value * allocValue(); - Env & allocEnv(size_t size); + inline Value * allocValue(); + inline Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); Value * allocAttr(Value & vAttrs, std::string_view name); @@ -512,3 +512,5 @@ extern EvalSettings evalSettings; static const std::string corepkgsPrefix{"/__corepkgs__/"}; } + +#include "eval-inline.hh" From 47baa9d43c0339b0a738b9b75c5ddcfb07d7131d Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 1 Jan 2022 19:24:20 +0100 Subject: [PATCH 11/41] make Pos smaller reduces peak hep memory use on eval of our test system from 264.4MB to 242.3MB, possibly also a slight performance boost. theoretically memory use could be cut down by another eight bytes per Pos on average by turning it into a tuple containing an index into a global base position table with row and column offsets, but that doesn't seem worth the effort at this point. --- src/libexpr/nixexpr.hh | 13 ++++++------- src/libutil/error.hh | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 12b54b8eb..4dbe31510 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -23,14 +23,13 @@ MakeError(RestrictedPathError, Error); struct Pos { - FileOrigin origin; Symbol file; - unsigned int line, column; - - Pos() : origin(foString), line(0), column(0) { } - Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column) - : origin(origin), file(file), line(line), column(column) { } - + uint32_t line; + FileOrigin origin:2; + uint32_t column:30; + Pos() : line(0), origin(foString), column(0) { }; + Pos(FileOrigin origin, const Symbol & file, uint32_t line, uint32_t column) + : file(file), line(line), origin(origin), column(column) { }; operator bool() const { return line != 0; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d55e1d701..bb43aa53b 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -53,6 +53,7 @@ typedef enum { lvlVomit } Verbosity; +/* adjust Pos::origin bit width when adding stuff here */ typedef enum { foFile, foStdin, From 8e2eaaaf69d9e216fce3ca6f7913bd0e2048e4b2 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 31 Dec 2021 00:50:23 +0100 Subject: [PATCH 12/41] make Finally more local no need for function<> with c++17 deduction. this saves allocations and virtual calls, but has the same semantics otherwise. not going through function has the side effect of giving compilers more insight into the cleanup code, so we need a few local warning disables. --- src/libutil/finally.hh | 7 +++---- src/libutil/util.cc | 15 ++++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh index 7760cfe9a..dee2e8d2f 100644 --- a/src/libutil/finally.hh +++ b/src/libutil/finally.hh @@ -1,14 +1,13 @@ #pragma once -#include - /* A trivial class to run a function at the end of a scope. */ +template class Finally { private: - std::function fun; + Fn fun; public: - Finally(std::function fun) : fun(fun) { } + Finally(Fn fun) : fun(std::move(fun)) { } ~Finally() { fun(); } }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index b833038a9..9f13d5f02 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -682,7 +682,14 @@ std::string drainFD(int fd, bool block, const size_t reserveSize) void drainFD(int fd, Sink & sink, bool block) { - int saved; + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } Finally finally([&]() { if (!block) { @@ -691,12 +698,6 @@ void drainFD(int fd, Sink & sink, bool block) } }); - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - std::vector buf(64 * 1024); while (1) { checkInterrupt(); From 4d629c4f7abbbe58dfe6d9d2b37541cdf2331606 Mon Sep 17 00:00:00 2001 From: pennae Date: Wed, 5 Jan 2022 01:48:26 +0100 Subject: [PATCH 13/41] add HAVE_BOEHMGC guards to batched allocation functions --- src/libexpr/eval-inline.hh | 13 +++++++++---- src/libexpr/eval.hh | 2 ++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 3331a7643..08a419923 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -42,6 +42,7 @@ inline void * allocBytes(size_t n) [[gnu::always_inline]] Value * EvalState::allocValue() { +#if HAVE_BOEHMGC /* We use the boehm batch allocator to speed up allocations of Values (of which there are many). GC_malloc_many returns a linked list of objects of the given size, where the first word of each object is also the pointer to the next object in the list. This also means that we @@ -56,6 +57,9 @@ Value * EvalState::allocValue() void * p = *valueAllocCache; *valueAllocCache = GC_NEXT(p); GC_NEXT(p) = nullptr; +#else + void * p = allocBytes(sizeof(Value)); +#endif nrValues++; return (Value *) p; @@ -70,9 +74,8 @@ Env & EvalState::allocEnv(size_t size) Env * env; - if (size != 1) - env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); - else { +#if HAVE_BOEHMGC + if (size == 1) { /* see allocValue for explanations. */ if (!*env1AllocCache) { *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *)); @@ -83,7 +86,9 @@ Env & EvalState::allocEnv(size_t size) *env1AllocCache = GC_NEXT(p); GC_NEXT(p) = nullptr; env = (Env *) p; - } + } else +#endif + env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); env->type = Env::Plain; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d2efe8a47..f1e00bae7 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -133,11 +133,13 @@ private: /* Cache used by prim_match(). */ std::shared_ptr regexCache; +#if HAVE_BOEHMGC /* Allocation cache for GC'd Value objects. */ std::shared_ptr valueAllocCache; /* Allocation cache for size-1 Env objects. */ std::shared_ptr env1AllocCache; +#endif public: From 4d98143914120d0163f5c50f30ce8a5289433f8f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Mar 2022 20:31:50 +0100 Subject: [PATCH 14/41] BuildResult: Remove unused drvPath field --- src/libstore/build-result.hh | 4 ++-- src/libstore/build/derivation-goal.cc | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index d39d23a76..cb6d19b8e 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -67,8 +67,8 @@ struct BuildResult /* The derivation we built or the store path we substituted. */ DerivedPath path; - /* For derivations, the derivation path and the wanted outputs. */ - std::optional drvPath; + /* For derivations, a mapping from the names of the wanted outputs + to actual paths. */ DrvOutputs builtOutputs; /* The start/stop times of the build (or one of the rounds, if it diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 325635e2e..afed9bf16 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1324,7 +1324,6 @@ void DerivationGoal::done( DrvOutputs builtOutputs, std::optional ex) { - buildResult.drvPath = drvPath; buildResult.status = status; if (ex) // FIXME: strip: "error: " From 167766b65ca203238f733efd0b5df92d28ab66c4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Mar 2022 11:19:21 +0100 Subject: [PATCH 15/41] Style --- src/libfetchers/git.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 6884e8de7..d75c5d3ae 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -239,13 +239,13 @@ struct GitInputScheme : InputScheme auto errorMessage = result.second; if (errorMessage.find("fatal: not a git repository") != std::string::npos) { - throw Error("'%s' is not a git repository.", actualUrl); + throw Error("'%s' is not a Git repository", actualUrl); } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { // indicates that the repo does not have any commits // we want to proceed and will consider it dirty later } else if (exitCode != 0) { // any other errors should lead to a failure - throw Error("Getting the HEAD of the git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage); + throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage); } bool hasHead = exitCode == 0; From 073e134de6260de7d90b135b7e173157741d4853 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 8 Mar 2022 17:45:19 +0000 Subject: [PATCH 16/41] Rename `requireGcStore` to `GcStore::require` I should have done this to begin with. This will be nicer once more Store sub-interfaces exist too, to illustrate the pattern. --- src/libstore/daemon.cc | 6 +++--- src/libstore/gc-store.cc | 2 +- src/libstore/gc-store.hh | 4 ++-- src/nix-collect-garbage/nix-collect-garbage.cc | 2 +- src/nix-store/nix-store.cc | 6 +++--- src/nix/store-delete.cc | 2 +- src/nix/store-gc.cc | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e6760664c..ffc605410 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -645,7 +645,7 @@ static void performOp(TunnelLogger * logger, ref store, Path path = absPath(readString(from)); logger->startWork(); - auto & gcStore = requireGcStore(*store); + auto & gcStore = GcStore::require(*store); gcStore.addIndirectRoot(path); logger->stopWork(); @@ -663,7 +663,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopFindRoots: { logger->startWork(); - auto & gcStore = requireGcStore(*store); + auto & gcStore = GcStore::require(*store); Roots roots = gcStore.findRoots(!trusted); logger->stopWork(); @@ -695,7 +695,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (options.ignoreLiveness) throw Error("you are not allowed to ignore liveness"); - auto & gcStore = requireGcStore(*store); + auto & gcStore = GcStore::require(*store); gcStore.collectGarbage(options, results); logger->stopWork(); diff --git a/src/libstore/gc-store.cc b/src/libstore/gc-store.cc index 3dbdec53b..4516cc744 100644 --- a/src/libstore/gc-store.cc +++ b/src/libstore/gc-store.cc @@ -2,7 +2,7 @@ namespace nix { -GcStore & requireGcStore(Store & store) +GcStore & GcStore::require(Store & store) { auto * gcStore = dynamic_cast(&store); if (!gcStore) diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 829f70dc4..462cc097b 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -77,8 +77,8 @@ struct GcStore : public virtual Store /* Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + + static GcStore & require(Store & store); }; -GcStore & requireGcStore(Store & store); - } diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 4b28ea6a4..b61f0e640 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -81,7 +81,7 @@ static int main_nix_collect_garbage(int argc, char * * argv) // Run the actual garbage collector. if (!dryRun) { auto store = openStore(); - auto & gcStore = requireGcStore(*store); + auto & gcStore = GcStore::require(*store); options.action = GCOptions::gcDeleteDead; GCResults results; PrintFreed freed(true, results); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 8ebaf9387..33493500c 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -429,7 +429,7 @@ static void opQuery(Strings opFlags, Strings opArgs) store->computeFSClosure( args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); - auto & gcStore = requireGcStore(*store); + auto & gcStore = GcStore::require(*store); Roots roots = gcStore.findRoots(false); for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) @@ -590,7 +590,7 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); - auto & gcStore = requireGcStore(*store); + auto & gcStore = GcStore::require(*store); if (printRoots) { Roots roots = gcStore.findRoots(false); @@ -629,7 +629,7 @@ static void opDelete(Strings opFlags, Strings opArgs) for (auto & i : opArgs) options.pathsToDelete.insert(store->followLinksToStorePath(i)); - auto & gcStore = requireGcStore(*store); + auto & gcStore = GcStore::require(*store); GCResults results; PrintFreed freed(true, results); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index aa7a8b12f..0de7efaba 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -33,7 +33,7 @@ struct CmdStoreDelete : StorePathsCommand void run(ref store, std::vector && storePaths) override { - auto & gcStore = requireGcStore(*store); + auto & gcStore = GcStore::require(*store); for (auto & path : storePaths) options.pathsToDelete.insert(path); diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 21718dc0c..515f4ffdb 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -34,7 +34,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun void run(ref store) override { - auto & gcStore = requireGcStore(*store); + auto & gcStore = GcStore::require(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; GCResults results; From 89effe9d4abde2ea8970736f79e0a6a499777692 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 8 Mar 2022 18:28:29 +0000 Subject: [PATCH 17/41] `GcStore::resolve` should print the URI --- src/libstore/gc-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/gc-store.cc b/src/libstore/gc-store.cc index 4516cc744..dc2b82166 100644 --- a/src/libstore/gc-store.cc +++ b/src/libstore/gc-store.cc @@ -6,7 +6,7 @@ GcStore & GcStore::require(Store & store) { auto * gcStore = dynamic_cast(&store); if (!gcStore) - throw UsageError("Garbage collection not supported by this store"); + throw UsageError("Garbage collection not supported by store '%s'", store.getUri()); return *gcStore; } From 678d1c2aa0f499466c723d3461277dc197515f57 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 8 Mar 2022 18:20:39 +0000 Subject: [PATCH 18/41] Factor out a `LogStore` interface Continue progress on #5729. Just as I hoped, this uncovered an issue: the daemon protocol is missing a way to query build logs. This doesn't effect `unix://`, but does effect `ssh://`. A FIXME is left for this, so we come back to it later. --- src/libstore/binary-cache-store.hh | 5 ++++- src/libstore/build/local-derivation-goal.cc | 6 ++++++ src/libstore/daemon.cc | 4 +++- src/libstore/local-fs-store.hh | 6 +++++- src/libstore/log-store.cc | 13 +++++++++++++ src/libstore/log-store.hh | 19 +++++++++++++++++++ src/libstore/remote-store.hh | 6 +++++- src/libstore/ssh-store.cc | 4 ++++ src/libstore/store-api.hh | 8 -------- src/nix-store/nix-store.cc | 9 ++++++--- src/nix/log.cc | 14 +++++++++++--- src/nix/repl.cc | 12 ++++++++++-- src/nix/store-copy-log.cc | 8 ++++++-- 13 files changed, 92 insertions(+), 22 deletions(-) create mode 100644 src/libstore/log-store.cc create mode 100644 src/libstore/log-store.hh diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 9603a8caa..ca538b3cb 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -2,6 +2,7 @@ #include "crypto.hh" #include "store-api.hh" +#include "log-store.hh" #include "pool.hh" @@ -28,7 +29,9 @@ struct BinaryCacheStoreConfig : virtual StoreConfig "other than -1 which we reserve to indicate Nix defaults should be used"}; }; -class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store +class BinaryCacheStore : public virtual BinaryCacheStoreConfig, + public virtual Store, + public virtual LogStore { private: diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a372728f5..4e763e570 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1340,6 +1340,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo next->queryMissing(allowed, willBuild, willSubstitute, unknown, downloadSize, narSize); } + + virtual std::optional getBuildLog(const StorePath & path) override + { return std::nullopt; } + + virtual void addBuildLog(const StorePath & path, std::string_view log) override + { unsupported("addBuildLog"); } }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ffc605410..1e24a1c79 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -4,6 +4,7 @@ #include "build-result.hh" #include "store-api.hh" #include "gc-store.hh" +#include "log-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" @@ -953,11 +954,12 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!trusted) throw Error("you are not privileged to add logs"); + auto & logStore = LogStore::require(*store); { FramedSource source(from); StringSink sink; source.drainInto(sink); - store->addBuildLog(path, sink.s); + logStore.addBuildLog(path, sink.s); } logger->stopWork(); to << 1; diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index fbd49dc2c..e6fb3201a 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -2,6 +2,7 @@ #include "store-api.hh" #include "gc-store.hh" +#include "log-store.hh" namespace nix { @@ -24,7 +25,10 @@ struct LocalFSStoreConfig : virtual StoreConfig "physical path to the Nix store"}; }; -class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore +class LocalFSStore : public virtual LocalFSStoreConfig, + public virtual Store, + public virtual GcStore, + public virtual LogStore { public: diff --git a/src/libstore/log-store.cc b/src/libstore/log-store.cc new file mode 100644 index 000000000..56aec91f2 --- /dev/null +++ b/src/libstore/log-store.cc @@ -0,0 +1,13 @@ +#include "log-store.hh" + +namespace nix { + +LogStore & LogStore::require(Store & store) +{ + auto * gcStore = dynamic_cast(&store); + if (!gcStore) + throw UsageError("Build log storage and retrieval not supported by store '%s'", store.getUri()); + return *gcStore; +} + +} diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh new file mode 100644 index 000000000..ad17cbe6e --- /dev/null +++ b/src/libstore/log-store.hh @@ -0,0 +1,19 @@ +#pragma once + +#include "store-api.hh" + + +namespace nix { + +struct LogStore : public virtual Store +{ + /* Return the build log of the specified store path, if available, + or null otherwise. */ + virtual std::optional getBuildLog(const StorePath & path) = 0; + + virtual void addBuildLog(const StorePath & path, std::string_view log) = 0; + + static LogStore & require(Store & store); +}; + +} diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9f6f50593..8493be6fc 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -5,6 +5,7 @@ #include "store-api.hh" #include "gc-store.hh" +#include "log-store.hh" namespace nix { @@ -30,7 +31,10 @@ struct RemoteStoreConfig : virtual StoreConfig /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore +class RemoteStore : public virtual RemoteStoreConfig, + public virtual Store, + public virtual GcStore, + public virtual LogStore { public: diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index bb03daef4..62daa838c 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -52,6 +52,10 @@ public: bool sameMachine() override { return false; } + // FIXME extend daemon protocol, move implementation to RemoteStore + std::optional getBuildLog(const StorePath & path) override + { unsupported("getBuildLog"); } + private: struct Connection : RemoteStore::Connection diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e99a3f2cb..635a82a2a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -605,14 +605,6 @@ public: */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); - /* Return the build log of the specified store path, if available, - or null otherwise. */ - virtual std::optional getBuildLog(const StorePath & path) - { return std::nullopt; } - - virtual void addBuildLog(const StorePath & path, std::string_view log) - { unsupported("addBuildLog"); } - /* Hack to allow long-running processes like hydra-queue-runner to occasionally flush their path info cache. */ void clearPathInfoCache() diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 33493500c..b3f28bcc2 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -4,6 +4,7 @@ #include "globals.hh" #include "build-result.hh" #include "gc-store.hh" +#include "log-store.hh" #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" @@ -474,13 +475,15 @@ static void opReadLog(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); + auto & logStore = LogStore::require(*store); + RunPager pager; for (auto & i : opArgs) { - auto path = store->followLinksToStorePath(i); - auto log = store->getBuildLog(path); + auto path = logStore.followLinksToStorePath(i); + auto log = logStore.getBuildLog(path); if (!log) - throw Error("build log of derivation '%s' is not available", store->printStorePath(path)); + throw Error("build log of derivation '%s' is not available", logStore.printStorePath(path)); std::cout << *log; } } diff --git a/src/nix/log.cc b/src/nix/log.cc index fd3c1d787..72d02ef11 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "log-store.hh" #include "progress-bar.hh" using namespace nix; @@ -34,17 +35,24 @@ struct CmdLog : InstallableCommand RunPager pager; for (auto & sub : subs) { + auto * logSubP = dynamic_cast(&*sub); + if (!logSubP) { + printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri()); + continue; + } + auto & logSub = *logSubP; + auto log = std::visit(overloaded { [&](const DerivedPath::Opaque & bo) { - return sub->getBuildLog(bo.path); + return logSub.getBuildLog(bo.path); }, [&](const DerivedPath::Built & bfd) { - return sub->getBuildLog(bfd.drvPath); + return logSub.getBuildLog(bfd.drvPath); }, }, b.raw()); if (!log) continue; stopProgressBar(); - printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); + printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); std::cout << *log; return; } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 3a51a13e6..916353d8c 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -25,6 +25,7 @@ extern "C" { #include "eval-inline.hh" #include "attr-path.hh" #include "store-api.hh" +#include "log-store.hh" #include "common-eval-args.hh" #include "get-drvs.hh" #include "derivations.hh" @@ -526,9 +527,16 @@ bool NixRepl::processLine(std::string line) bool foundLog = false; RunPager pager; for (auto & sub : subs) { - auto log = sub->getBuildLog(drvPath); + auto * logSubP = dynamic_cast(&*sub); + if (!logSubP) { + printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri()); + continue; + } + auto & logSub = *logSubP; + + auto log = logSub.getBuildLog(drvPath); if (log) { - printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri()); + printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.getUri()); logger->writeToStdout(*log); foundLog = true; break; diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 079cd6b3e..22b3980c0 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "log-store.hh" #include "sync.hh" #include "thread-pool.hh" @@ -26,7 +27,10 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand void run(ref srcStore) override { + auto & srcLogStore = LogStore::require(*srcStore); + auto dstStore = getDstStore(); + auto & dstLogStore = LogStore::require(*dstStore); StorePathSet drvPaths; @@ -35,8 +39,8 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand drvPaths.insert(drvPath); for (auto & drvPath : drvPaths) { - if (auto log = srcStore->getBuildLog(drvPath)) - dstStore->addBuildLog(drvPath, *log); + if (auto log = srcLogStore.getBuildLog(drvPath)) + dstLogStore.addBuildLog(drvPath, *log); else throw Error("build log for '%s' is not available", srcStore->printStorePath(drvPath)); } From a03b1fd7f60788304f358d5f4dc063c7c9e650a9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 9 Mar 2022 15:27:48 +0000 Subject: [PATCH 19/41] Deduplicate the Store downcasting with a template --- src/libstore/daemon.cc | 9 +++++---- src/libstore/gc-store.cc | 13 ------------- src/libstore/gc-store.hh | 4 ++-- src/libstore/log-store.cc | 13 ------------- src/libstore/log-store.hh | 2 ++ src/libstore/store-cast.hh | 16 ++++++++++++++++ src/nix-collect-garbage/nix-collect-garbage.cc | 3 ++- src/nix-store/nix-store.cc | 9 +++++---- src/nix/store-copy-log.cc | 5 +++-- src/nix/store-delete.cc | 3 ++- src/nix/store-gc.cc | 3 ++- 11 files changed, 39 insertions(+), 41 deletions(-) delete mode 100644 src/libstore/gc-store.cc delete mode 100644 src/libstore/log-store.cc create mode 100644 src/libstore/store-cast.hh diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 1e24a1c79..9f21ecf36 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -3,6 +3,7 @@ #include "worker-protocol.hh" #include "build-result.hh" #include "store-api.hh" +#include "store-cast.hh" #include "gc-store.hh" #include "log-store.hh" #include "path-with-outputs.hh" @@ -646,7 +647,7 @@ static void performOp(TunnelLogger * logger, ref store, Path path = absPath(readString(from)); logger->startWork(); - auto & gcStore = GcStore::require(*store); + auto & gcStore = require(*store); gcStore.addIndirectRoot(path); logger->stopWork(); @@ -664,7 +665,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopFindRoots: { logger->startWork(); - auto & gcStore = GcStore::require(*store); + auto & gcStore = require(*store); Roots roots = gcStore.findRoots(!trusted); logger->stopWork(); @@ -696,7 +697,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (options.ignoreLiveness) throw Error("you are not allowed to ignore liveness"); - auto & gcStore = GcStore::require(*store); + auto & gcStore = require(*store); gcStore.collectGarbage(options, results); logger->stopWork(); @@ -954,7 +955,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!trusted) throw Error("you are not privileged to add logs"); - auto & logStore = LogStore::require(*store); + auto & logStore = require(*store); { FramedSource source(from); StringSink sink; diff --git a/src/libstore/gc-store.cc b/src/libstore/gc-store.cc deleted file mode 100644 index dc2b82166..000000000 --- a/src/libstore/gc-store.cc +++ /dev/null @@ -1,13 +0,0 @@ -#include "gc-store.hh" - -namespace nix { - -GcStore & GcStore::require(Store & store) -{ - auto * gcStore = dynamic_cast(&store); - if (!gcStore) - throw UsageError("Garbage collection not supported by store '%s'", store.getUri()); - return *gcStore; -} - -} diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 462cc097b..b3cbbad74 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -61,6 +61,8 @@ struct GCResults struct GcStore : public virtual Store { + inline static std::string operationName = "Garbage collection"; + /* Add an indirect root, which is merely a symlink to `path' from /nix/var/nix/gcroots/auto/. `path' is supposed to be a symlink to a store path. The garbage collector will @@ -77,8 +79,6 @@ struct GcStore : public virtual Store /* Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; - - static GcStore & require(Store & store); }; } diff --git a/src/libstore/log-store.cc b/src/libstore/log-store.cc deleted file mode 100644 index 56aec91f2..000000000 --- a/src/libstore/log-store.cc +++ /dev/null @@ -1,13 +0,0 @@ -#include "log-store.hh" - -namespace nix { - -LogStore & LogStore::require(Store & store) -{ - auto * gcStore = dynamic_cast(&store); - if (!gcStore) - throw UsageError("Build log storage and retrieval not supported by store '%s'", store.getUri()); - return *gcStore; -} - -} diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index ad17cbe6e..ff1b92e17 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -7,6 +7,8 @@ namespace nix { struct LogStore : public virtual Store { + inline static std::string operationName = "Build log storage and retrieval"; + /* Return the build log of the specified store path, if available, or null otherwise. */ virtual std::optional getBuildLog(const StorePath & path) = 0; diff --git a/src/libstore/store-cast.hh b/src/libstore/store-cast.hh new file mode 100644 index 000000000..ff62fc359 --- /dev/null +++ b/src/libstore/store-cast.hh @@ -0,0 +1,16 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +template +T & require(Store & store) +{ + auto * castedStore = dynamic_cast(&store); + if (!castedStore) + throw UsageError("%s not supported by store '%s'", T::operationName, store.getUri()); + return *castedStore; +} + +} diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index b61f0e640..af6f1c88c 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,4 +1,5 @@ #include "store-api.hh" +#include "store-cast.hh" #include "gc-store.hh" #include "profiles.hh" #include "shared.hh" @@ -81,7 +82,7 @@ static int main_nix_collect_garbage(int argc, char * * argv) // Run the actual garbage collector. if (!dryRun) { auto store = openStore(); - auto & gcStore = GcStore::require(*store); + auto & gcStore = require(*store); options.action = GCOptions::gcDeleteDead; GCResults results; PrintFreed freed(true, results); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b3f28bcc2..153b84137 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -3,6 +3,7 @@ #include "dotgraph.hh" #include "globals.hh" #include "build-result.hh" +#include "store-cast.hh" #include "gc-store.hh" #include "log-store.hh" #include "local-store.hh" @@ -430,7 +431,7 @@ static void opQuery(Strings opFlags, Strings opArgs) store->computeFSClosure( args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); - auto & gcStore = GcStore::require(*store); + auto & gcStore = require(*store); Roots roots = gcStore.findRoots(false); for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) @@ -475,7 +476,7 @@ static void opReadLog(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - auto & logStore = LogStore::require(*store); + auto & logStore = require(*store); RunPager pager; @@ -593,7 +594,7 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); - auto & gcStore = GcStore::require(*store); + auto & gcStore = require(*store); if (printRoots) { Roots roots = gcStore.findRoots(false); @@ -632,7 +633,7 @@ static void opDelete(Strings opFlags, Strings opArgs) for (auto & i : opArgs) options.pathsToDelete.insert(store->followLinksToStorePath(i)); - auto & gcStore = GcStore::require(*store); + auto & gcStore = require(*store); GCResults results; PrintFreed freed(true, results); diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 22b3980c0..2e288f743 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "store-cast.hh" #include "log-store.hh" #include "sync.hh" #include "thread-pool.hh" @@ -27,10 +28,10 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand void run(ref srcStore) override { - auto & srcLogStore = LogStore::require(*srcStore); + auto & srcLogStore = require(*srcStore); auto dstStore = getDstStore(); - auto & dstLogStore = LogStore::require(*dstStore); + auto & dstLogStore = require(*dstStore); StorePathSet drvPaths; diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 0de7efaba..ca43f1530 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "store-cast.hh" #include "gc-store.hh" using namespace nix; @@ -33,7 +34,7 @@ struct CmdStoreDelete : StorePathsCommand void run(ref store, std::vector && storePaths) override { - auto & gcStore = GcStore::require(*store); + auto & gcStore = require(*store); for (auto & path : storePaths) options.pathsToDelete.insert(path); diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 515f4ffdb..8b9b5d164 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "store-cast.hh" #include "gc-store.hh" using namespace nix; @@ -34,7 +35,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun void run(ref store) override { - auto & gcStore = GcStore::require(*store); + auto & gcStore = require(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; GCResults results; From 015d2ad507959b06987ba7ed4566689aba316766 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 11 Mar 2022 14:42:09 +0000 Subject: [PATCH 20/41] Desugar `StorePathWithOutputs` in nix-build implementation `DerivedPath` has replaced `StorePathWithOutputs` internally, so shrinking the usage of `StorePathWithOutputs` to just the boundary is good. --- src/nix-build/nix-build.cc | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 795a4f7bd..faa8c078f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -325,8 +325,7 @@ static void main_nix_build(int argc, char * * argv) state->printStats(); - auto buildPaths = [&](const std::vector & paths0) { - auto paths = toDerivedPaths(paths0); + auto buildPaths = [&](const std::vector & paths) { /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ uint64_t downloadSize, narSize; @@ -348,7 +347,7 @@ static void main_nix_build(int argc, char * * argv) auto & drvInfo = drvs.front(); auto drv = evalStore->derivationFromPath(drvInfo.requireDrvPath()); - std::vector pathsToBuild; + std::vector pathsToBuild; RealisedPath::Set pathsToCopy; /* Figure out what bash shell to use. If $NIX_BUILD_SHELL @@ -370,7 +369,10 @@ static void main_nix_build(int argc, char * * argv) throw Error("the 'bashInteractive' attribute in did not evaluate to a derivation"); auto bashDrv = drv->requireDrvPath(); - pathsToBuild.push_back({bashDrv}); + pathsToBuild.push_back(DerivedPath::Built { + .drvPath = bashDrv, + .outputs = {}, + }); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; @@ -382,17 +384,24 @@ static void main_nix_build(int argc, char * * argv) } // Build or fetch all dependencies of the derivation. - for (const auto & input : drv.inputDrvs) + for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) { + // To get around lambda capturing restrictions in the + // standard. + const auto & inputDrv = inputDrv0; if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const std::string & exclude) { - return !std::regex_search(store->printStorePath(input.first), std::regex(exclude)); + return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude)); })) { - pathsToBuild.push_back({input.first, input.second}); - pathsToCopy.insert(input.first); + pathsToBuild.push_back(DerivedPath::Built { + .drvPath = inputDrv, + .outputs = inputOutputs + }); + pathsToCopy.insert(inputDrv); } + } for (const auto & src : drv.inputSrcs) { - pathsToBuild.push_back({src}); + pathsToBuild.push_back(DerivedPath::Opaque{src}); pathsToCopy.insert(src); } @@ -543,7 +552,7 @@ static void main_nix_build(int argc, char * * argv) else { - std::vector pathsToBuild; + std::vector pathsToBuild; std::vector> pathsToBuildOrdered; RealisedPath::Set drvsToCopy; @@ -556,7 +565,7 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back({drvPath, {outputName}}); + pathsToBuild.push_back(DerivedPath::Built{drvPath, {outputName}}); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); From 0948b8e94dfbf87ed2a695c6c7d8dac250e2c293 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Oct 2021 18:05:53 +0000 Subject: [PATCH 21/41] Reduce variants for derivation hash modulo This changes was taken from dynamic derivation (#4628). It` somewhat undoes the refactors I first did for floating CA derivations, as the benefit of hindsight + requirements of dynamic derivations made me reconsider some things. They aren't to consequential, but I figured they might be good to land first, before the more profound changes @thufschmitt has in the works. --- src/libexpr/primops.cc | 45 ++++++++------- src/libstore/derivations.cc | 111 ++++++++++++++++++++++-------------- src/libstore/derivations.hh | 40 +++++++++++-- src/libstore/local-store.cc | 4 +- src/nix/develop.cc | 3 +- 5 files changed, 131 insertions(+), 72 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3124025aa..0980e75f4 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1207,32 +1207,37 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * // hash per output. auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); std::visit(overloaded { - [&](Hash & h) { - for (auto & i : outputs) { - auto outPath = state.store->makeOutputPath(i, h, drvName); - drv.env[i] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = std::move(outPath), - }, - }); + [&](const DrvHash & drvHash) { + auto & h = drvHash.hash; + switch (drvHash.kind) { + case DrvHash::Kind::Deferred: + for (auto & i : outputs) { + drv.outputs.insert_or_assign(i, + DerivationOutput { + .output = DerivationOutputDeferred{}, + }); + } + break; + case DrvHash::Kind::Regular: + for (auto & i : outputs) { + auto outPath = state.store->makeOutputPath(i, h, drvName); + drv.env[i] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign(i, + DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(outPath), + }, + }); + } + break; } }, - [&](CaOutputHashes &) { + [&](const CaOutputHashes &) { // Shouldn't happen as the toplevel derivation is not CA. assert(false); }, - [&](DeferredHash &) { - for (auto & i : outputs) { - drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputDeferred{}, - }); - } - }, }, - hashModulo); + hashModulo.raw()); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a49be0057..1fe45bd87 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -510,7 +510,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & */ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - bool isDeferred = false; + auto kind = DrvHash::Kind::Regular; /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { case DerivationType::CAFixed: { @@ -526,7 +526,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m return outputHashes; } case DerivationType::CAFloating: - isDeferred = true; + kind = DrvHash::Kind::Deferred; break; case DerivationType::InputAddressed: break; @@ -537,21 +537,20 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* For other derivations, replace the inputs paths with recursive calls to this function. */ std::map inputs2; - for (auto & i : drv.inputDrvs) { - const auto & res = pathDerivationModulo(store, i.first); + for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { + // Avoid lambda capture restriction with standard / Clang + auto & inputOutputs = inputOutputs0; + const auto & res = pathDerivationModulo(store, drvPath); std::visit(overloaded { // Regular non-CA derivation, replace derivation - [&](const Hash & drvHash) { - inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); - }, - [&](const DeferredHash & deferredHash) { - isDeferred = true; - inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second); + [&](const DrvHash & drvHash) { + kind |= drvHash.kind; + inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs); }, // CA derivation's output hashes [&](const CaOutputHashes & outputHashes) { std::set justOut = { "out" }; - for (auto & output : i.second) { + for (auto & output : inputOutputs) { /* Put each one in with a single "out" output.. */ const auto h = outputHashes.at(output); inputs2.insert_or_assign( @@ -559,15 +558,24 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m justOut); } }, - }, res); + }, res.raw()); } auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); - if (isDeferred) - return DeferredHash { hash }; - else - return hash; + return DrvHash { .hash = hash, .kind = kind }; +} + + +void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept +{ + switch (other) { + case DrvHash::Kind::Regular: + break; + case DrvHash::Kind::Deferred: + self = other; + break; + } } @@ -575,20 +583,15 @@ std::map staticOutputHashes(Store & store, const Derivation & { std::map res; std::visit(overloaded { - [&](const Hash & drvHash) { + [&](const DrvHash & drvHash) { for (auto & outputName : drv.outputNames()) { - res.insert({outputName, drvHash}); - } - }, - [&](const DeferredHash & deferredHash) { - for (auto & outputName : drv.outputNames()) { - res.insert({outputName, deferredHash.hash}); + res.insert({outputName, drvHash.hash}); } }, [&](const CaOutputHashes & outputHashes) { res = outputHashes; }, - }, hashDerivationModulo(store, drv, true)); + }, hashDerivationModulo(store, drv, true).raw()); return res; } @@ -738,7 +741,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { if (std::holds_alternative(output.output)) { - Hash h = std::get(hashModulo); + auto & h = hashModulo.requireNoFixedNonDeferred(); auto outPath = store.makeOutputPath(outputName, h, drv.name); drv.env[outputName] = store.printStorePath(outPath); output = DerivationOutput { @@ -751,31 +754,51 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } +const Hash & DrvHashModulo::requireNoFixedNonDeferred() const { + auto * drvHashOpt = std::get_if(&raw()); + assert(drvHashOpt); + assert(drvHashOpt->kind == DrvHash::Kind::Regular); + return drvHashOpt->hash; +} + +static bool tryResolveInput( + Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, + const StorePath & inputDrv, const StringSet & inputOutputs) +{ + auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv); + + auto getOutput = [&](const std::string & outputName) { + auto & actualPathOpt = inputDrvOutputs.at(outputName); + if (!actualPathOpt) + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(inputDrv) + ); + return actualPathOpt; + }; + + for (auto & outputName : inputOutputs) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + inputRewrites.emplace( + downstreamPlaceholder(store, inputDrv, outputName), + store.printStorePath(actualPath)); + inputSrcs.insert(std::move(actualPath)); + } + + return true; +} + std::optional Derivation::tryResolve(Store & store) { BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & input : inputDrvs) { - auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first); - StringSet newOutputNames; - for (auto & outputName : input.second) { - auto actualPathOpt = inputDrvOutputs.at(outputName); - if (!actualPathOpt) { - warn("output %s of input %s missing, aborting the resolving", - outputName, - store.printStorePath(input.first) - ); - return std::nullopt; - } - auto actualPath = *actualPathOpt; - inputRewrites.emplace( - downstreamPlaceholder(store, input.first, outputName), - store.printStorePath(actualPath)); - resolved.inputSrcs.insert(std::move(actualPath)); - } - } + for (auto & [inputDrv, inputOutputs] : inputDrvs) + if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs)) + return std::nullopt; rewriteDerivation(store, resolved, inputRewrites); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 132de82b6..2fb18d7f7 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -175,13 +175,43 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName // whose output hashes are always known since they are fixed up-front. typedef std::map CaOutputHashes; -struct DeferredHash { Hash hash; }; +struct DrvHash { + Hash hash; + + enum struct Kind { + // Statically determined derivations. + // This hash will be directly used to compute the output paths + Regular, + // Floating-output derivations (and their dependencies). + Deferred, + }; + + Kind kind; +}; + +void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; typedef std::variant< - Hash, // regular DRV normalized hash - CaOutputHashes, // Fixed-output derivation hashes - DeferredHash // Deferred hashes for floating outputs drvs and their dependencies -> DrvHashModulo; + // Regular normalized derivation hash, and whether it was deferred (because + // an ancestor derivation is a floating content addressed derivation). + DrvHash, + // Fixed-output derivation hashes + CaOutputHashes +> DrvHashModuloRaw; + +struct DrvHashModulo : DrvHashModuloRaw { + using Raw = DrvHashModuloRaw; + using Raw::Raw; + + /* Get hash, throwing if it is per-output CA hashes or a + deferred Drv hash. + */ + const Hash & requireNoFixedNonDeferred() const; + + inline const Raw & raw() const { + return static_cast(*this); + } +}; /* Returns hashes with the details of fixed-output subderivations expunged. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1ee71b1c0..9230be15a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -701,8 +701,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat [&](const DerivationOutputInputAddressed & doia) { if (!h) { // somewhat expensive so we do lazily - auto temp = hashDerivationModulo(*this, drv, true); - h = std::get(temp); + auto h0 = hashDerivationModulo(*this, drv, true); + h = h0.requireNoFixedNonDeferred(); } StorePath recomputed = makeOutputPath(i.first, *h, drvName); if (doia.path != recomputed) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8af5da9d0..dafcafd79 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -206,7 +206,8 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; drv.env[output.first] = ""; } - Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true)); + auto h0 = hashDerivationModulo(*evalStore, drv, true); + const Hash & h = h0.requireNoFixedNonDeferred(); for (auto & output : drv.outputs) { auto outPath = store->makeOutputPath(output.first, h, drv.name); From cb1a76112eb77883038ebad8d3b0e39330bdf1b3 Mon Sep 17 00:00:00 2001 From: Artturin Date: Sat, 12 Mar 2022 23:58:29 +0200 Subject: [PATCH 22/41] nix-env: Add a suggestion for when there's a name collision in channels help new users find a solution to their problem ./result/bin/nix-env -qa hello warning: name collision in input Nix expressions, skipping '/home/artturin/.nix-defexpr/channels_root/master' suggestion: remove 'master' from either the root channels or the user channels hello-2.12 hello-2.12 --- src/nix-env/nix-env.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 40c3c5d65..eb5fbc08f 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -128,7 +128,12 @@ static void getAllExprs(EvalState & state, if (hasSuffix(attrName, ".nix")) attrName = std::string(attrName, 0, attrName.size() - 4); if (!seen.insert(attrName).second) { - printError("warning: name collision in input Nix expressions, skipping '%1%'", path2); + std::string suggestionMessage = ""; + if (path2.find("channels") != std::string::npos && path.find("channels") != std::string::npos) { + suggestionMessage = fmt("\nsuggestion: remove '%s' from either the root channels or the user channels", attrName); + } + printError("warning: name collision in input Nix expressions, skipping '%1%'" + "%2%", path2, suggestionMessage); continue; } /* Load the expression on demand. */ From 6b1872312f1b505334acb67b8bf7990b0a0fdbd8 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Sat, 12 Mar 2022 21:56:08 +0000 Subject: [PATCH 23/41] nix store gc: account for auto-optimised store Before the change on a system with `auto-optimise-store = true`: $ nix store gc --verbose --max 1 deleted all the paths instead of one path (we requested 1 byte limit). It happens because every file in `auto-optimise-store = true` has at least 2 links: file itself and a link in /nix/store/.links/ directory. The change conservatively assumes that any file that has one (as before) or two links (assume auto-potimise mode) will free space. Co-authored-by: Sandro --- src/libstore/gc.cc | 3 ++- src/libutil/util.cc | 25 +++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 024da66c1..69755bd19 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -841,7 +841,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (unlink(path.c_str()) == -1) throw SysError("deleting '%1%'", path); - results.bytesFreed += st.st_size; + /* Do not accound for deleted file here. Rely on deletePath() + accounting. */ } struct stat st; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9f13d5f02..70eaf4f9c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -406,8 +406,29 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) throw SysError("getting status of '%1%'", path); } - if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_size; + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } if (S_ISDIR(st.st_mode)) { /* Make the directory accessible. */ From e06b264f944226cfede0b3fc0394a9c3bbde4a78 Mon Sep 17 00:00:00 2001 From: thkoch2001 <94641+thkoch2001@users.noreply.github.com> Date: Sun, 13 Mar 2022 18:42:26 +0200 Subject: [PATCH 24/41] Add documentation= entry to systemd unit file Closes: #6246 --- misc/systemd/nix-daemon.service.in | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 25655204d..b4badf2ba 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -1,5 +1,6 @@ [Unit] Description=Nix Daemon +Documentation=man:nix-daemon https://nixos.org/manual RequiresMountsFor=@storedir@ RequiresMountsFor=@localstatedir@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket From 34e20c164cd512c10af99803215e32889a6d965b Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 18 Feb 2022 16:56:35 +0100 Subject: [PATCH 25/41] libfetchers/path: set `lastModified` to path's mtime When importing e.g. a local `nixpkgs` in a flake to test a change like { inputs.nixpkgs.url = path:/home/ma27/Projects/nixpkgs; outputs = /* ... */ } then the input is missing a `lastModified`-field that's e.g. used in `nixpkgs.lib.nixosSystem`. Due to the missing `lastMoified`-field, the mtime is set to 19700101: result -> /nix/store/b7dg1lmmsill2rsgyv2w7b6cnmixkvc1-nixos-system-nixos-22.05.19700101.dirty With this change, the `path`-fetcher now sets a `lastModified` attribute to the `mtime` just like it's the case in the `tarball`-fetcher already. When building NixOS systems with `nixpkgs` being a `path`-input and this patch, the output-path now looks like this: result -> /nix/store/ld2qf9c1s98dxmiwcaq5vn9k5ylzrm1s-nixos-system-nixos-22.05.20220217.dirty --- src/libfetchers/path.cc | 15 ++++++++++++--- src/libutil/archive.cc | 19 +++++++++++++++---- src/libutil/archive.hh | 4 ++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 59e228e97..2d5d7d58d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -1,5 +1,7 @@ #include "fetchers.hh" #include "store-api.hh" +#include +#include "archive.hh" namespace nix::fetchers { @@ -80,8 +82,9 @@ struct PathInputScheme : InputScheme // nothing to do } - std::pair fetch(ref store, const Input & input) override + std::pair fetch(ref store, const Input & _input) override { + Input input(_input); std::string absPath; auto path = getStrAttr(input.attrs, "path"); @@ -111,9 +114,15 @@ struct PathInputScheme : InputScheme if (storePath) store->addTempRoot(*storePath); - if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) + time_t mtime = 0; + if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { // FIXME: try to substitute storePath. - storePath = store->addToStore("source", absPath); + auto src = sinkToSource([&](Sink & sink) { + mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter); + }); + storePath = store->addToStoreFromDump(*src, "source"); + } + input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); return {std::move(*storePath), input}; } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index eda004756..30b471af5 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -64,11 +64,12 @@ static void dumpContents(const Path & path, off_t size, } -static void dump(const Path & path, Sink & sink, PathFilter & filter) +static time_t dump(const Path & path, Sink & sink, PathFilter & filter) { checkInterrupt(); auto st = lstat(path); + time_t result = st.st_mtime; sink << "("; @@ -103,7 +104,10 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) for (auto & i : unhacked) if (filter(path + "/" + i.first)) { sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + "/" + i.second, sink, filter); + auto tmp_mtime = dump(path + "/" + i.second, sink, filter); + if (tmp_mtime > result) { + result = tmp_mtime; + } sink << ")"; } } @@ -114,13 +118,20 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) else throw Error("file '%1%' has an unsupported type", path); sink << ")"; + + return result; } -void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { sink << narVersionMagic1; - dump(path, sink, filter); + return dump(path, sink, filter); +} + +void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +{ + dumpPathAndGetMtime(path, sink, filter); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index fca351605..79ce08df0 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -48,6 +48,10 @@ namespace nix { void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +/* Same as `void dumpPath()`, but returns the last modified date of the path */ +time_t dumpPathAndGetMtime(const Path & path, Sink & sink, + PathFilter & filter = defaultPathFilter); + void dumpString(std::string_view s, Sink & sink); /* FIXME: fix this API, it sucks. */ From 244baff2c7bc287b83377f1ed807a861fc705dac Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 28 Feb 2022 11:56:50 +0100 Subject: [PATCH 26/41] libfetchers: remove obsolete filesystem #include --- src/libfetchers/path.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 2d5d7d58d..f0ef97da5 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -1,6 +1,5 @@ #include "fetchers.hh" #include "store-api.hh" -#include #include "archive.hh" namespace nix::fetchers { From 975bade7f0adb0d122eabeeca6ad44bf1d1f4366 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 15 Mar 2022 12:55:32 +0100 Subject: [PATCH 27/41] Implement simple test for `path`-fetcher setting a correct `lastModifiedDate` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- tests/fetchPath.sh | 6 ++++++ tests/local.mk | 1 + 2 files changed, 7 insertions(+) create mode 100644 tests/fetchPath.sh diff --git a/tests/fetchPath.sh b/tests/fetchPath.sh new file mode 100644 index 000000000..0cfdf68cf --- /dev/null +++ b/tests/fetchPath.sh @@ -0,0 +1,6 @@ +source common.sh + +touch foo -t 222211111111 +# We only check whether 2222-11-1* **:**:** is the last modified date since +# `lastModified` is transformed into UTC in `builtins.fetchTarball`. +[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$PWD/foo\").lastModifiedDate")" =~ 2222111.* ]] diff --git a/tests/local.mk b/tests/local.mk index 8032fc38a..b69fb9391 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -21,6 +21,7 @@ nix_tests = \ tarball.sh \ fetchGit.sh \ fetchurl.sh \ + fetchPath.sh \ simple.sh \ referrers.sh \ optimise-store.sh \ From 2d5c43f210694033eb0d1ddfb0131e18a159b790 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Mar 2022 21:05:01 +0100 Subject: [PATCH 28/41] Fix the tests on 32bits machines year 2222 is too much for a 32 bit timestamp. So replace it by something smaller --- tests/fetchPath.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fetchPath.sh b/tests/fetchPath.sh index 0cfdf68cf..0e065cae2 100644 --- a/tests/fetchPath.sh +++ b/tests/fetchPath.sh @@ -1,6 +1,6 @@ source common.sh -touch foo -t 222211111111 +touch foo -t 202211111111 # We only check whether 2222-11-1* **:**:** is the last modified date since # `lastModified` is transformed into UTC in `builtins.fetchTarball`. -[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$PWD/foo\").lastModifiedDate")" =~ 2222111.* ]] +[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$PWD/foo\").lastModifiedDate")" =~ 2022111.* ]] From 3cea6f569e300c92d23e46eb4e2039302f58518f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 16 Mar 2022 08:56:01 +0100 Subject: [PATCH 29/41] =?UTF-8?q?Fix=20the=20date=20in=20the=20comment=20o?= =?UTF-8?q?f=20fetchPath=E2=80=99s=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: pennae <82953136+pennae@users.noreply.github.com> --- tests/fetchPath.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetchPath.sh b/tests/fetchPath.sh index 0e065cae2..8f17638e9 100644 --- a/tests/fetchPath.sh +++ b/tests/fetchPath.sh @@ -1,6 +1,6 @@ source common.sh touch foo -t 202211111111 -# We only check whether 2222-11-1* **:**:** is the last modified date since +# We only check whether 2022-11-1* **:**:** is the last modified date since # `lastModified` is transformed into UTC in `builtins.fetchTarball`. [[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$PWD/foo\").lastModifiedDate")" =~ 2022111.* ]] From af013281c9212e49709ef4ecb429a4307f48bea0 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Wed, 16 Mar 2022 12:53:38 +0100 Subject: [PATCH 30/41] distributed-builds.md: fixing typo of the most minor sort --- doc/manual/src/advanced-topics/distributed-builds.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index c4c60db15..b0d7fbf1a 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -110,7 +110,7 @@ default, set it to `-`. 7. A comma-separated list of *mandatory features*. A machine will only be used to build a derivation if all of the machine’s mandatory features appear in the derivation’s `requiredSystemFeatures` - attribute.. + attribute. 8. The (base64-encoded) public host key of the remote machine. If omitted, SSH will use its regular known-hosts file. Specifically, the field is calculated From a5c969db496843b9a42b3aee494a02714ee9ca78 Mon Sep 17 00:00:00 2001 From: Artturin Date: Wed, 16 Mar 2022 20:48:50 +0200 Subject: [PATCH 31/41] nix: allow using --file - to read from stdin --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libcmd/installables.cc | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2f..c9753f9aa 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,3 @@ # Release X.Y (202?-??-??) + +* Various nix commands can now read expressions from stdin with `--file -`. diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index b7623d4ba..784117569 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -134,7 +134,9 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "file", .shortName = 'f', - .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.", + .description = + "Interpret installables as attribute paths relative to the Nix expression stored in *file*. " + "If *file* is the character -, then a Nix expression will be read from standard input.", .category = installablesCategory, .labels = {"file"}, .handler = {&file}, @@ -695,7 +697,10 @@ std::vector> SourceExprCommand::parseInstallables( auto state = getEvalState(); auto vFile = state->allocValue(); - if (file) + if (file == "-") { + auto e = state->parseStdin(); + state->eval(e, *vFile); + } else if (file) state->evalFile(lookupFileArg(*state, *file), *vFile); else { auto e = state->parseExprFromString(*expr, absPath(".")); From 4f8ad41d4eb2012f01cd09ad745e78c5b8d8bee6 Mon Sep 17 00:00:00 2001 From: Artturin Date: Wed, 16 Mar 2022 20:57:18 +0200 Subject: [PATCH 32/41] add tests for nix eval and nix-instantiate --- tests/eval.nix | 5 +++++ tests/eval.sh | 29 +++++++++++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 35 insertions(+) create mode 100644 tests/eval.nix create mode 100644 tests/eval.sh diff --git a/tests/eval.nix b/tests/eval.nix new file mode 100644 index 000000000..befbd17a9 --- /dev/null +++ b/tests/eval.nix @@ -0,0 +1,5 @@ +{ + int = 123; + str = "foo"; + attr.foo = "bar"; +} diff --git a/tests/eval.sh b/tests/eval.sh new file mode 100644 index 000000000..2e5ceb969 --- /dev/null +++ b/tests/eval.sh @@ -0,0 +1,29 @@ +source common.sh + +clearStore + +testStdinHeredoc=$(nix eval -f - < Date: Sat, 12 Mar 2022 13:47:01 +0100 Subject: [PATCH 33/41] nix-env: always print output names in JSON and XML The current `--out-path` flag has two disadvantages when one is only concerned with querying the names of outputs: - it requires evaluating every output's `outPath`, which takes significantly more resources and runs into more failures - it destroys the information of the order of outputs so we can't tell which one is the main output This patch makes the output names always present (replacing paths with `null` in JSON if `--out-path` isn't given), and adds an `outputName` field. --- src/libexpr/get-drvs.cc | 28 +++++---- src/libexpr/get-drvs.hh | 7 ++- src/nix-env/nix-env.cc | 136 +++++++++++++++++++++------------------- src/nix-env/user-env.cc | 10 +-- 4 files changed, 95 insertions(+), 86 deletions(-) diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 7630c5ff4..7f2ecb4f7 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -102,7 +102,7 @@ StorePath DrvInfo::queryOutPath() const } -DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) +DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall) { if (outputs.empty()) { /* Get the ‘outputs’ list. */ @@ -112,20 +112,24 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) /* For each output... */ for (auto elem : i->value->listItems()) { - /* Evaluate the corresponding set. */ - std::string name(state->forceStringNoCtx(*elem, *i->pos)); - Bindings::iterator out = attrs->find(state->symbols.create(name)); - if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value, *i->pos); + std::string output(state->forceStringNoCtx(*elem, *i->pos)); - /* And evaluate its ‘outPath’ attribute. */ - Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); - if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; - outputs.emplace(name, state->coerceToStorePath(*outPath->pos, *outPath->value, context)); + if (withPaths) { + /* Evaluate the corresponding set. */ + Bindings::iterator out = attrs->find(state->symbols.create(output)); + if (out == attrs->end()) continue; // FIXME: throw error? + state->forceAttrs(*out->value, *i->pos); + + /* And evaluate its ‘outPath’ attribute. */ + Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); + if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? + PathSet context; + outputs.emplace(output, state->coerceToStorePath(*outPath->pos, *outPath->value, context)); + } else + outputs.emplace(output, std::nullopt); } } else - outputs.emplace("out", queryOutPath()); + outputs.emplace("out", withPaths ? std::optional{queryOutPath()} : std::nullopt); } if (!onlyOutputsToInstall || !attrs) return outputs; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 3ca6f1fca..7cc1abef2 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -13,7 +13,7 @@ namespace nix { struct DrvInfo { public: - typedef std::map Outputs; + typedef std::map> Outputs; private: EvalState * state; @@ -46,8 +46,9 @@ public: StorePath requireDrvPath() const; StorePath queryOutPath() const; std::string queryOutputName() const; - /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ - Outputs queryOutputs(bool onlyOutputsToInstall = false); + /** Return the unordered map of output names to (optional) output paths. + * The "outputs to install" are determined by `meta.outputsToInstall`. */ + Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false); StringSet queryMetaNames(); Value * queryMeta(const std::string & name); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 40c3c5d65..e68218d1c 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -918,12 +918,17 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin pkgObj.attr("pname", drvName.name); pkgObj.attr("version", drvName.version); pkgObj.attr("system", i.querySystem()); + pkgObj.attr("outputName", i.queryOutputName()); - if (printOutPath) { - DrvInfo::Outputs outputs = i.queryOutputs(); + { + DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); JSONObject outputObj = pkgObj.object("outputs"); - for (auto & j : outputs) - outputObj.attr(j.first, globals.state->store->printStorePath(j.second)); + for (auto & j : outputs) { + if (j.second) + outputObj.attr(j.first, globals.state->store->printStorePath(*j.second)); + else + outputObj.attr(j.first, nullptr); + } } if (printMeta) { @@ -1154,13 +1159,16 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) columns.push_back(drvPath ? store.printStorePath(*drvPath) : "-"); } + if (xmlOutput) + attrs["outputName"] = i.queryOutputName(); + if (printOutPath && !xmlOutput) { DrvInfo::Outputs outputs = i.queryOutputs(); std::string s; for (auto & j : outputs) { if (!s.empty()) s += ';'; if (j.first != "out") { s += j.first; s += "="; } - s += store.printStorePath(j.second); + s += store.printStorePath(*j.second); } columns.push_back(s); } @@ -1174,71 +1182,67 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } if (xmlOutput) { - if (printOutPath || printMeta) { - XMLOpenElement item(xml, "item", attrs); - if (printOutPath) { - DrvInfo::Outputs outputs = i.queryOutputs(); - for (auto & j : outputs) { - XMLAttrs attrs2; - attrs2["name"] = j.first; - attrs2["path"] = store.printStorePath(j.second); - xml.writeEmptyElement("output", attrs2); - } - } - if (printMeta) { - StringSet metaNames = i.queryMetaNames(); - for (auto & j : metaNames) { - XMLAttrs attrs2; - attrs2["name"] = j; - Value * v = i.queryMeta(j); - if (!v) - printError( - "derivation '%s' has invalid meta attribute '%s'", - i.queryName(), j); - else { - if (v->type() == nString) { - attrs2["type"] = "string"; - attrs2["value"] = v->string.s; - xml.writeEmptyElement("meta", attrs2); - } else if (v->type() == nInt) { - attrs2["type"] = "int"; - attrs2["value"] = (format("%1%") % v->integer).str(); - xml.writeEmptyElement("meta", attrs2); - } else if (v->type() == nFloat) { - attrs2["type"] = "float"; - attrs2["value"] = (format("%1%") % v->fpoint).str(); - xml.writeEmptyElement("meta", attrs2); - } else if (v->type() == nBool) { - attrs2["type"] = "bool"; - attrs2["value"] = v->boolean ? "true" : "false"; - xml.writeEmptyElement("meta", attrs2); - } else if (v->type() == nList) { - attrs2["type"] = "strings"; - XMLOpenElement m(xml, "meta", attrs2); - for (auto elem : v->listItems()) { - if (elem->type() != nString) continue; - XMLAttrs attrs3; - attrs3["value"] = elem->string.s; - xml.writeEmptyElement("string", attrs3); - } - } else if (v->type() == nAttrs) { - attrs2["type"] = "strings"; - XMLOpenElement m(xml, "meta", attrs2); - Bindings & attrs = *v->attrs; - for (auto &i : attrs) { - Attr & a(*attrs.find(i.name)); - if(a.value->type() != nString) continue; - XMLAttrs attrs3; - attrs3["type"] = i.name; - attrs3["value"] = a.value->string.s; - xml.writeEmptyElement("string", attrs3); + XMLOpenElement item(xml, "item", attrs); + DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); + for (auto & j : outputs) { + XMLAttrs attrs2; + attrs2["name"] = j.first; + if (j.second) + attrs2["path"] = store.printStorePath(*j.second); + xml.writeEmptyElement("output", attrs2); + } + if (printMeta) { + StringSet metaNames = i.queryMetaNames(); + for (auto & j : metaNames) { + XMLAttrs attrs2; + attrs2["name"] = j; + Value * v = i.queryMeta(j); + if (!v) + printError( + "derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j); + else { + if (v->type() == nString) { + attrs2["type"] = "string"; + attrs2["value"] = v->string.s; + xml.writeEmptyElement("meta", attrs2); + } else if (v->type() == nInt) { + attrs2["type"] = "int"; + attrs2["value"] = (format("%1%") % v->integer).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type() == nFloat) { + attrs2["type"] = "float"; + attrs2["value"] = (format("%1%") % v->fpoint).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type() == nBool) { + attrs2["type"] = "bool"; + attrs2["value"] = v->boolean ? "true" : "false"; + xml.writeEmptyElement("meta", attrs2); + } else if (v->type() == nList) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + for (auto elem : v->listItems()) { + if (elem->type() != nString) continue; + XMLAttrs attrs3; + attrs3["value"] = elem->string.s; + xml.writeEmptyElement("string", attrs3); } - } + } else if (v->type() == nAttrs) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + Bindings & attrs = *v->attrs; + for (auto &i : attrs) { + Attr & a(*attrs.find(i.name)); + if(a.value->type() != nString) continue; + XMLAttrs attrs3; + attrs3["type"] = i.name; + attrs3["value"] = a.value->string.s; + xml.writeEmptyElement("string", attrs3); + } } } } - } else - xml.writeEmptyElement("item", attrs); + } } else table.push_back(columns); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index af4f350ff..bcc7736ac 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -56,7 +56,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, output paths, and optionally the derivation path, as well as the meta attributes. */ std::optional drvPath = keepDerivations ? i.queryDrvPath() : std::nullopt; - DrvInfo::Outputs outputs = i.queryOutputs(true); + DrvInfo::Outputs outputs = i.queryOutputs(true, true); StringSet metaNames = i.queryMetaNames(); auto attrs = state.buildBindings(7 + outputs.size()); @@ -76,15 +76,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, for (const auto & [m, j] : enumerate(outputs)) { (vOutputs.listElems()[m] = state.allocValue())->mkString(j.first); auto outputAttrs = state.buildBindings(2); - outputAttrs.alloc(state.sOutPath).mkString(state.store->printStorePath(j.second)); + outputAttrs.alloc(state.sOutPath).mkString(state.store->printStorePath(*j.second)); attrs.alloc(j.first).mkAttrs(outputAttrs); /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ - state.store->addTempRoot(j.second); - state.store->ensurePath(j.second); + state.store->addTempRoot(*j.second); + state.store->ensurePath(*j.second); - references.insert(j.second); + references.insert(*j.second); } // Copy the meta attributes. From 8dcecc07387e6a782b4831dcc0d846eceb396191 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 16 Mar 2022 16:15:11 +0100 Subject: [PATCH 34/41] nix-env: print a final newline after JSON --- src/nix-env/nix-env.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e68218d1c..a1cd8fd40 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1057,6 +1057,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Print the desired columns, or XML output. */ if (jsonOutput) { queryJSON(globals, elems, printOutPath, printMeta); + cout << '\n'; return; } From c20e07763d84c9dc6f8d06993335c765ba514aca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 16 Mar 2022 18:39:46 +0100 Subject: [PATCH 35/41] Add some tests for `nix-env -q --json` --- tests/user-envs.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/user-envs.sh b/tests/user-envs.sh index 430688de1..d63fe780a 100644 --- a/tests/user-envs.sh +++ b/tests/user-envs.sh @@ -17,6 +17,16 @@ outPath10=$(nix-env -f ./user-envs.nix -qa --out-path --no-name '*' | grep foo-1 drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1.0) [ -n "$outPath10" -a -n "$drvPath10" ] +# Query with json +nix-env -f ./user-envs.nix -qa --json | jq -e '.[] | select(.name == "bar-0.1") | [ + .outputName == "out", + .outputs.out == null +] | all' +nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name == "bar-0.1") | [ + .outputName == "out", + (.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1")) +] | all' + # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly rm -rf $HOME/.nix-defexpr From 3fc4c612fbde332d66b78dcc5b17b7d0d5235484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 11:34:31 +0100 Subject: [PATCH 36/41] Fix `nix build --dry-run` with CA derivations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don’t try and assume that we know the output paths when we’ve just built with `--dry-run`. Instead make `--dry-run` follow a different code path that won’t assume the knowledge of the output paths at all. Fix #6275 --- src/libstore/derived-path.cc | 31 ++++++++++++++++++++++++++----- src/libstore/derived-path.hh | 2 ++ src/nix/build.cc | 17 ++++++++++++++--- tests/build-dry.sh | 19 +++++++++++++++++++ tests/ca/build-dry.sh | 6 ++++++ 5 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 tests/ca/build-dry.sh diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 194489580..0183bda35 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -11,6 +11,21 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { return res; } +nlohmann::json DerivedPath::Built::toJSON(ref store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); + for (const auto& output : outputs) { + if (knownOutputs.at(output)) + res["outputs"][output] = store->printStorePath(knownOutputs.at(output).value()); + else + res["outputs"][output] = nullptr; + } + return res; +} + nlohmann::json BuiltPath::Built::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); @@ -35,16 +50,22 @@ StorePathSet BuiltPath::outPaths() const ); } -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) { +template +nlohmann::json stuffToJSON(const std::vector & ts, ref store) { auto res = nlohmann::json::array(); - for (const BuiltPath & buildable : buildables) { - std::visit([&res, store](const auto & buildable) { - res.push_back(buildable.toJSON(store)); - }, buildable.raw()); + for (const T & t : ts) { + std::visit([&res, store](const auto & t) { + res.push_back(t.toJSON(store)); + }, t.raw()); } return res; } +nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) +{ return stuffToJSON(buildables, store); } +nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) +{ return stuffToJSON(paths, store); } + std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 9d6ace069..8ca0882a4 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -45,6 +45,7 @@ struct DerivedPathBuilt { std::string to_string(const Store & store) const; static DerivedPathBuilt parse(const Store & store, std::string_view); + nlohmann::json toJSON(ref store) const; }; using _DerivedPathRaw = std::variant< @@ -119,5 +120,6 @@ typedef std::vector DerivedPaths; typedef std::vector BuiltPaths; nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store); +nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref store); } diff --git a/src/nix/build.cc b/src/nix/build.cc index 680db1c60..840c7ca38 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -52,15 +52,26 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile void run(ref store) override { + if (dryRun) { + std::vector pathsToBuild; + + for (auto & i : installables) { + auto b = i->toDerivedPaths(); + pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); + } + printMissing(store, pathsToBuild, lvlError); + if (json) + logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + return; + } + auto buildables = Installable::build( getEvalStore(), store, - dryRun ? Realise::Derivation : Realise::Outputs, + Realise::Outputs, installables, buildMode); if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); - if (dryRun) return; - if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) for (const auto & [_i, buildable] : enumerate(buildables)) { diff --git a/tests/build-dry.sh b/tests/build-dry.sh index e72533e70..f0f38e9a0 100644 --- a/tests/build-dry.sh +++ b/tests/build-dry.sh @@ -50,3 +50,22 @@ nix build -f dependencies.nix -o $RESULT --dry-run nix build -f dependencies.nix -o $RESULT [[ -h $RESULT ]] + +################################################### +# Check the JSON output +clearStore +clearCache + +RES=$(nix build -f dependencies.nix --dry-run --json) + +if [[ -z "$NIX_TESTS_CA_BY_DEFAULT" ]]; then + echo "$RES" | jq '.[0] | [ + (.drvPath | test("'$NIX_STORE_DIR'.*\\.drv")), + (.outputs.out | test("'$NIX_STORE_DIR'")) + ] | all' +else + echo "$RES" | jq '.[0] | [ + (.drvPath | test("'$NIX_STORE_DIR'.*\\.drv")), + .outputs.out == null + ] | all' +fi diff --git a/tests/ca/build-dry.sh b/tests/ca/build-dry.sh new file mode 100644 index 000000000..9a72075ec --- /dev/null +++ b/tests/ca/build-dry.sh @@ -0,0 +1,6 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. && source build-dry.sh + From d58453f72ea584cac2e3362fd6a73fcf0e3b615e Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Thu, 17 Mar 2022 18:36:45 +0000 Subject: [PATCH 37/41] gc: don't visit implicit referrers on garbage collection Before the change garbage collector was not considering `.drv` and outputs as alive even if configuration says otherwise. As a result `nix store gc --dry-run` could visit (and parse) `.drv` files multiple times (worst case it's quadratic). It happens because `alive` set was populating only runtime closure without regard for actual configuration. The change fixes it. Benchmark: my system has about 139MB, 40K `.drv` files. Performance before the change: $ time nix store gc --dry-run real 4m22,148s Performance after the change: $ time nix store gc --dry-run real 0m14,178s --- src/libstore/gc.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 69755bd19..f65fb1b2e 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -678,7 +678,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) alive.insert(start); try { StorePathSet closure; - computeFSClosure(*path, closure); + computeFSClosure(*path, closure, + /* flipDirection */ false, gcKeepOutputs, gcKeepDerivations); for (auto & p : closure) alive.insert(p); } catch (InvalidPath &) { } From 197feed51dd770929ba8f6f12aec50ed8597a747 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 17 Mar 2022 22:29:15 +0000 Subject: [PATCH 38/41] Clean up `DerivationOutput`, and headers 1. `DerivationOutput` now as the `std::variant` as a base class. And the variants are given hierarchical names under `DerivationOutput`. In 8e0d0689be797f9e42f9b43b06f50c1af7f20b4a @matthewbauer and I didn't know a better idiom, and so we made it a field. But this sort of "newtype" is anoying for literals downstream. Since then we leaned the base class, inherit the constructors trick, e.g. used in `DerivedPath`. Switching to use that makes this more ergonomic, and consistent. 2. `store-api.hh` and `derivations.hh` are now independent. In bcde5456cc3295061a0726881c3e441444dd6680 I swapped the dependency, but I now know it is better to just keep on using incomplete types as much as possible for faster compilation and good separation of concerns. --- src/libexpr/get-drvs.cc | 1 + src/libexpr/primops.cc | 36 ++++------ src/libexpr/primops/context.cc | 1 + src/libstore/build/local-derivation-goal.cc | 14 ++-- src/libstore/derivations.cc | 80 +++++++++------------ src/libstore/derivations.hh | 35 +++++---- src/libstore/derived-path.cc | 1 + src/libstore/local-store.cc | 10 +-- src/libstore/misc.cc | 2 +- src/libstore/parsed-derivations.hh | 1 + src/libstore/repair-flag.hh | 7 ++ src/libstore/store-api.cc | 1 + src/libstore/store-api.hh | 4 +- src/nix/app.cc | 1 + src/nix/develop.cc | 12 ++-- src/nix/show-derivation.cc | 10 +-- 16 files changed, 113 insertions(+), 103 deletions(-) create mode 100644 src/libstore/repair-flag.hh diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 7f2ecb4f7..bb7e77b61 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,6 +1,7 @@ #include "get-drvs.hh" #include "util.hh" #include "eval-inline.hh" +#include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0980e75f4..4f9b6f0e0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1163,26 +1163,24 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput { - .output = DerivationOutputCAFixed { - .hash = FixedOutputHash { - .method = ingestionMethod, - .hash = std::move(h), - }, + drv.outputs.insert_or_assign("out", + DerivationOutput::CAFixed { + .hash = FixedOutputHash { + .method = ingestionMethod, + .hash = std::move(h), }, - }); + }); } else if (contentAddressed) { HashType ht = parseHashType(outputHashAlgo); for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); - drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputCAFloating { + drv.outputs.insert_or_assign(i, + DerivationOutput::CAFloating { .method = ingestionMethod, .hashType = ht, - }, - }); + }); } } @@ -1196,10 +1194,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : outputs) { drv.env[i] = ""; drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = StorePath::dummy, - }, + DerivationOutput::InputAddressed { + .path = StorePath::dummy, }); } @@ -1213,9 +1209,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * case DrvHash::Kind::Deferred: for (auto & i : outputs) { drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputDeferred{}, - }); + DerivationOutput::Deferred { }); } break; case DrvHash::Kind::Regular: @@ -1223,10 +1217,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeOutputPath(i, h, drvName); drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = std::move(outPath), - }, + DerivationOutput::InputAddressed { + .path = std::move(outPath), }); } break; diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 3701bd442..324394d10 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "derivations.hh" #include "store-api.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4e763e570..8a301d4ac 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2279,7 +2279,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() return res; }; - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { + auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { auto & st = outputStats.at(outputName); if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ @@ -2346,7 +2346,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() ValidPathInfo newInfo = std::visit(overloaded { - [&](const DerivationOutputInputAddressed & output) { + [&](const DerivationOutput::InputAddressed & output) { /* input-addressed case */ auto requiredFinalPath = output.path; /* Preemptively add rewrite rule for final hash, as that is @@ -2366,8 +2366,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() return newInfo0; }, - [&](const DerivationOutputCAFixed & dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { + [&](const DerivationOutput::CAFixed & dof) { + auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { .method = dof.hash.method, .hashType = dof.hash.hash.type, }); @@ -2389,17 +2389,17 @@ DrvOutputs LocalDerivationGoal::registerOutputs() return newInfo0; }, - [&](DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { return newInfoFromCA(dof); }, - [&](DerivationOutputDeferred) -> ValidPathInfo { + [&](const DerivationOutput::Deferred &) -> ValidPathInfo { // No derivation should reach that point without having been // rewritten first assert(false); }, - }, output.output); + }, output.raw()); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1fe45bd87..3dd3a78d8 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -11,25 +11,25 @@ namespace nix { std::optional DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return std::visit(overloaded { - [](const DerivationOutputInputAddressed & doi) -> std::optional { + [](const DerivationOutput::InputAddressed & doi) -> std::optional { return { doi.path }; }, - [&](const DerivationOutputCAFixed & dof) -> std::optional { + [&](const DerivationOutput::CAFixed & dof) -> std::optional { return { dof.path(store, drvName, outputName) }; }, - [](const DerivationOutputCAFloating & dof) -> std::optional { + [](const DerivationOutput::CAFloating & dof) -> std::optional { return std::nullopt; }, - [](const DerivationOutputDeferred &) -> std::optional { + [](const DerivationOutput::Deferred &) -> std::optional { return std::nullopt; }, - }, output); + }, raw()); } -StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { +StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( hash.method, hash.hash, outputPathName(drvName, outputName)); @@ -179,35 +179,27 @@ static DerivationOutput parseDerivationOutput(const Store & store, const auto hashType = parseHashType(hashAlgo); if (hash != "") { validatePath(pathS); - return DerivationOutput { - .output = DerivationOutputCAFixed { - .hash = FixedOutputHash { - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(hash, hashType), - }, + return DerivationOutput::CAFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed(hash, hashType), }, }; } else { settings.requireExperimentalFeature(Xp::CaDerivations); assert(pathS == ""); - return DerivationOutput { - .output = DerivationOutputCAFloating { - .method = std::move(method), - .hashType = std::move(hashType), - }, + return DerivationOutput::CAFloating { + .method = std::move(method), + .hashType = std::move(hashType), }; } } else { if (pathS == "") { - return DerivationOutput { - .output = DerivationOutputDeferred { } - }; + return DerivationOutput::Deferred { }; } validatePath(pathS); - return DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = store.parseStorePath(pathS), - } + return DerivationOutput::InputAddressed { + .path = store.parseStorePath(pathS), }; } } @@ -335,27 +327,27 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); std::visit(overloaded { - [&](const DerivationOutputInputAddressed & doi) { + [&](const DerivationOutput::InputAddressed & doi) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path)); s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); }, - [&](const DerivationOutputCAFixed & dof) { + [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); }, - [&](const DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, - [&](const DerivationOutputDeferred &) { + [&](const DerivationOutput::Deferred &) { s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); } - }, i.second.output); + }, i.second.raw()); s += ')'; } @@ -423,13 +415,13 @@ DerivationType BasicDerivation::type() const std::optional floatingHashType; for (auto & i : outputs) { std::visit(overloaded { - [&](const DerivationOutputInputAddressed &) { + [&](const DerivationOutput::InputAddressed &) { inputAddressedOutputs.insert(i.first); }, - [&](const DerivationOutputCAFixed &) { + [&](const DerivationOutput::CAFixed &) { fixedCAOutputs.insert(i.first); }, - [&](const DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { floatingCAOutputs.insert(i.first); if (!floatingHashType) { floatingHashType = dof.hashType; @@ -438,10 +430,10 @@ DerivationType BasicDerivation::type() const throw Error("All floating outputs must use the same hash type"); } }, - [&](const DerivationOutputDeferred &) { + [&](const DerivationOutput::Deferred &) { deferredIAOutputs.insert(i.first); }, - }, i.second.output); + }, i.second.raw()); } if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { @@ -516,7 +508,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { - auto & dof = std::get(i.second.output); + auto & dof = std::get(i.second.raw()); auto hash = hashString(htSHA256, "fixed:out:" + dof.hash.printMethodAlgo() + ":" + dof.hash.hash.to_string(Base16, false) + ":" @@ -672,27 +664,27 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr for (auto & i : drv.outputs) { out << i.first; std::visit(overloaded { - [&](const DerivationOutputInputAddressed & doi) { + [&](const DerivationOutput::InputAddressed & doi) { out << store.printStorePath(doi.path) << "" << ""; }, - [&](const DerivationOutputCAFixed & dof) { + [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.hash.printMethodAlgo() << dof.hash.hash.to_string(Base16, false); }, - [&](const DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { out << "" << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, - [&](const DerivationOutputDeferred &) { + [&](const DerivationOutput::Deferred &) { out << "" << "" << ""; }, - }, i.second.output); + }, i.second.raw()); } worker_proto::write(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; @@ -740,14 +732,12 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative(output.output)) { + if (std::holds_alternative(output.raw())) { auto & h = hashModulo.requireNoFixedNonDeferred(); auto outPath = store.makeOutputPath(outputName, h, drv.name); drv.env[outputName] = store.printStorePath(outPath); - output = DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = std::move(outPath), - }, + output = DerivationOutput::InputAddressed { + .path = std::move(outPath), }; } } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 2fb18d7f7..6b92b2f21 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "hash.hh" #include "content-address.hh" +#include "repair-flag.hh" #include "sync.hh" #include @@ -44,19 +45,31 @@ struct DerivationOutputCAFloating */ struct DerivationOutputDeferred {}; -struct DerivationOutput +typedef std::variant< + DerivationOutputInputAddressed, + DerivationOutputCAFixed, + DerivationOutputCAFloating, + DerivationOutputDeferred +> _DerivationOutputRaw; + +struct DerivationOutput : _DerivationOutputRaw { - std::variant< - DerivationOutputInputAddressed, - DerivationOutputCAFixed, - DerivationOutputCAFloating, - DerivationOutputDeferred - > output; + using Raw = _DerivationOutputRaw; + using Raw::Raw; + + using InputAddressed = DerivationOutputInputAddressed; + using CAFixed = DerivationOutputCAFixed; + using CAFloating = DerivationOutputCAFloating; + using Deferred = DerivationOutputDeferred; /* Note, when you use this function you should make sure that you're passing the right derivation name. When in doubt, you should use the safer interface provided by BasicDerivation::outputsAndOptPaths */ std::optional path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + inline const Raw & raw() const { + return static_cast(*this); + } }; typedef std::map DerivationOutputs; @@ -150,8 +163,6 @@ struct Derivation : BasicDerivation class Store; -enum RepairFlag : bool { NoRepair = false, Repair = true }; - /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(Store & store, const Derivation & drv, @@ -197,10 +208,10 @@ typedef std::variant< DrvHash, // Fixed-output derivation hashes CaOutputHashes -> DrvHashModuloRaw; +> _DrvHashModuloRaw; -struct DrvHashModulo : DrvHashModuloRaw { - using Raw = DrvHashModuloRaw; +struct DrvHashModulo : _DrvHashModuloRaw { + using Raw = _DrvHashModuloRaw; using Raw::Raw; /* Get hash, throwing if it is per-output CA hashes or a diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 0183bda35..319b1c790 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,4 +1,5 @@ #include "derived-path.hh" +#include "derivations.hh" #include "store-api.hh" #include diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9230be15a..75726a1ab 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -698,7 +698,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat std::optional h; for (auto & i : drv.outputs) { std::visit(overloaded { - [&](const DerivationOutputInputAddressed & doia) { + [&](const DerivationOutput::InputAddressed & doia) { if (!h) { // somewhat expensive so we do lazily auto h0 = hashDerivationModulo(*this, drv, true); @@ -710,16 +710,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); envHasRightPath(doia.path, i.first); }, - [&](const DerivationOutputCAFixed & dof) { + [&](const DerivationOutput::CAFixed & dof) { StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); envHasRightPath(path, i.first); }, - [&](const DerivationOutputCAFloating &) { + [&](const DerivationOutput::CAFloating &) { /* Nothing to check */ }, - [&](const DerivationOutputDeferred &) { + [&](const DerivationOutput::Deferred &) { }, - }, i.second.output); + }, i.second.raw()); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 6409874ff..1f0bae7fe 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -87,7 +87,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out != drv.outputs.end()) { - if (auto v = std::get_if(&out->second.output)) + if (const auto * v = std::get_if(&out->second.raw())) return v->hash; } return std::nullopt; diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index effcf099d..95bec21e8 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,5 +1,6 @@ #pragma once +#include "derivations.hh" #include "store-api.hh" #include diff --git a/src/libstore/repair-flag.hh b/src/libstore/repair-flag.hh new file mode 100644 index 000000000..a13cda312 --- /dev/null +++ b/src/libstore/repair-flag.hh @@ -0,0 +1,7 @@ +#pragma once + +namespace nix { + +enum RepairFlag : bool { NoRepair = false, Repair = true }; + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 86fa6a211..59937be4d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,6 +1,7 @@ #include "crypto.hh" #include "fs-accessor.hh" #include "globals.hh" +#include "derivations.hh" #include "store-api.hh" #include "util.hh" #include "nar-info-disk-cache.hh" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 635a82a2a..0c8a4db56 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -10,8 +10,8 @@ #include "sync.hh" #include "globals.hh" #include "config.hh" -#include "derivations.hh" #include "path-info.hh" +#include "repair-flag.hh" #include #include @@ -62,6 +62,8 @@ MakeError(BadStorePath, Error); MakeError(InvalidStoreURI, Error); +struct BasicDerivation; +struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; diff --git a/src/nix/app.cc b/src/nix/app.cc index 2563180fb..30e138a96 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -4,6 +4,7 @@ #include "eval-cache.hh" #include "names.hh" #include "command.hh" +#include "derivations.hh" namespace nix { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index dafcafd79..6baf539c3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -196,14 +196,14 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore drv.inputSrcs.insert(std::move(getEnvShPath)); if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { for (auto & output : drv.outputs) { - output.second = { - .output = DerivationOutputDeferred{}, - }; + output.second = DerivationOutput::Deferred {}, drv.env[output.first] = hashPlaceholder(output.first); } } else { for (auto & output : drv.outputs) { - output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; + output.second = DerivationOutput::InputAddressed { + .path = StorePath::dummy, + }; drv.env[output.first] = ""; } auto h0 = hashDerivationModulo(*evalStore, drv, true); @@ -211,7 +211,9 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore for (auto & output : drv.outputs) { auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; + output.second = DerivationOutput::InputAddressed { + .path = outPath, + }; drv.env[output.first] = store->printStorePath(outPath); } } diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 61a02c9b3..0d9655732 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -65,19 +65,19 @@ struct CmdShowDerivation : InstallablesCommand auto & outputName = _outputName; // work around clang bug auto outputObj { outputsObj.object(outputName) }; std::visit(overloaded { - [&](const DerivationOutputInputAddressed & doi) { + [&](const DerivationOutput::InputAddressed & doi) { outputObj.attr("path", store->printStorePath(doi.path)); }, - [&](const DerivationOutputCAFixed & dof) { + [&](const DerivationOutput::CAFixed & dof) { outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); }, - [&](const DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, - [&](const DerivationOutputDeferred &) {}, - }, output.output); + [&](const DerivationOutput::Deferred &) {}, + }, output.raw()); } } From 8496be7defed3584c9fcd3b5d905fb84a9515d6f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 18 Mar 2022 02:07:31 +0000 Subject: [PATCH 39/41] Use Deferred when building an input-addressed drv Easier than using dummy path with input addressed. --- src/libexpr/primops.cc | 22 ++++++++-------------- src/nix/develop.cc | 4 +--- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4f9b6f0e0..489bee826 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1194,9 +1194,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : outputs) { drv.env[i] = ""; drv.outputs.insert_or_assign(i, - DerivationOutput::InputAddressed { - .path = StorePath::dummy, - }); + DerivationOutput::Deferred { }); } // Regular, non-CA derivation should always return a single hash and not @@ -1207,19 +1205,15 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto & h = drvHash.hash; switch (drvHash.kind) { case DrvHash::Kind::Deferred: - for (auto & i : outputs) { - drv.outputs.insert_or_assign(i, - DerivationOutput::Deferred { }); - } + /* Outputs already deferred, nothing to do */ break; case DrvHash::Kind::Regular: - for (auto & i : outputs) { - auto outPath = state.store->makeOutputPath(i, h, drvName); - drv.env[i] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign(i, - DerivationOutput::InputAddressed { - .path = std::move(outPath), - }); + for (auto & [outputName, output] : drv.outputs) { + auto outPath = state.store->makeOutputPath(outputName, h, drvName); + drv.env[outputName] = state.store->printStorePath(outPath); + output = DerivationOutput::InputAddressed { + .path = std::move(outPath), + }; } break; } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 6baf539c3..d2f9b5a6a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -201,9 +201,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore } } else { for (auto & output : drv.outputs) { - output.second = DerivationOutput::InputAddressed { - .path = StorePath::dummy, - }; + output.second = DerivationOutput::Deferred { }; drv.env[output.first] = ""; } auto h0 = hashDerivationModulo(*evalStore, drv, true); From 049fae155a18784ca59d194bf3e579fadbc3b48f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 18 Mar 2022 02:15:32 +0000 Subject: [PATCH 40/41] Avoid some pointless copying of drvs --- src/libexpr/primops.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 489bee826..2dc33103d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1199,7 +1199,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * // Regular, non-CA derivation should always return a single hash and not // hash per output. - auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); + auto hashModulo = hashDerivationModulo(*state.store, drv, true); std::visit(overloaded { [&](const DrvHash & drvHash) { auto & h = drvHash.hash; @@ -1240,7 +1240,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * However, we don't bother doing this for floating CA derivations because their "hash modulo" is indeterminate until built. */ if (drv.type() != DerivationType::CAFloating) { - auto h = hashDerivationModulo(*state.store, Derivation(drv), false); + auto h = hashDerivationModulo(*state.store, drv, false); drvHashes.lock()->insert_or_assign(drvPath, h); } From a544ed7684bdf5fa3c0b78d40913f5be3f73f5a7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 18 Mar 2022 00:36:52 +0000 Subject: [PATCH 41/41] Generalize `DerivationType` in preparation for impure derivations --- src/build-remote/build-remote.cc | 2 +- src/libexpr/primops.cc | 7 +- src/libstore/build/derivation-goal.cc | 35 ++++-- src/libstore/build/local-derivation-goal.cc | 14 +-- src/libstore/daemon.cc | 8 +- src/libstore/derivations.cc | 112 +++++++++++--------- src/libstore/derivations.hh | 62 +++++++---- src/libstore/local-store.cc | 1 + src/libstore/parsed-derivations.cc | 2 +- 9 files changed, 148 insertions(+), 95 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 8c9133c17..ff8ba2724 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -300,7 +300,7 @@ connected: std::set missingRealisations; StorePathSet missingPaths; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2dc33103d..bfa18f898 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1235,11 +1235,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Optimisation, but required in read-only mode! because in that case we don't actually write store derivations, so we can't - read them later. - - However, we don't bother doing this for floating CA derivations because - their "hash modulo" is indeterminate until built. */ - if (drv.type() != DerivationType::CAFloating) { + read them later. */ + { auto h = hashDerivationModulo(*state.store, drv, false); drvHashes.lock()->insert_or_assign(drvPath, h); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index afed9bf16..40c445836 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -204,7 +204,7 @@ void DerivationGoal::haveDerivation() { trace("have derivation"); - if (drv->type() == DerivationType::CAFloating) + if (!drv->type().hasKnownOutputPaths()) settings.requireExperimentalFeature(Xp::CaDerivations); retrySubstitution = false; @@ -440,9 +440,28 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && - ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) - || fullDrv.type() == DerivationType::DeferredInputAddressed)) { + auto drvType = fullDrv.type(); + bool resolveDrv = std::visit(overloaded { + [&](const DerivationType::InputAddressed & ia) { + /* must resolve if deferred. */ + return ia.deferred; + }, + [&](const DerivationType::ContentAddressed & ca) { + return !fullDrv.inputDrvs.empty() && ( + ca.fixed + /* Can optionally resolve if fixed, which is good + for avoiding unnecessary rebuilds. */ + ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + /* Must resolve if floating and there are any inputs + drvs. */ + : true); + }, + }, drvType.raw()); + + if (resolveDrv) + { + settings.requireExperimentalFeature(Xp::CaDerivations); + /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ @@ -501,7 +520,7 @@ void DerivationGoal::inputsRealised() /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; + nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -908,7 +927,7 @@ void DerivationGoal::buildDone() st = dynamic_cast(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : + derivationType.isImpure() || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -1221,7 +1240,7 @@ void DerivationGoal::flushLine() std::map> DerivationGoal::queryPartialDerivationOutputMap() { - if (!useDerivation || drv->type() != DerivationType::CAFloating) { + if (!useDerivation || drv->type().hasKnownOutputPaths()) { std::map> res; for (auto & [name, output] : drv->outputs) res.insert_or_assign(name, output.path(worker.store, drv->name, name)); @@ -1233,7 +1252,7 @@ std::map> DerivationGoal::queryPartialDeri OutputPathMap DerivationGoal::queryDerivationOutputMap() { - if (!useDerivation || drv->type() != DerivationType::CAFloating) { + if (!useDerivation || drv->type().hasKnownOutputPaths()) { OutputPathMap res; for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) res.insert_or_assign(name, *output.second); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8a301d4ac..75e7e6ca3 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; + useChroot = !(derivationType.isImpure()) && !noChroot; } auto & localStore = getLocalStore(); @@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder() "nogroup:x:65534:\n", sandboxGid())); /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) + if (!(derivationType.isImpure())) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder() us. */ - if (!(derivationIsImpure(derivationType))) + if (!(derivationType.isImpure())) privateNetwork = true; userNamespaceSync.create(); @@ -1049,7 +1049,7 @@ void LocalDerivationGoal::initEnv() derivation, tell the builder, so that for instance `fetchurl' can skip checking the output. On older Nixes, this environment variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; + if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; /* *Only* if this is a fixed-output derivation, propagate the values of the environment variables specified in the @@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { + if (derivationType.isImpure()) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) env[i] = getEnv(i).value_or(""); } @@ -1674,7 +1674,7 @@ void LocalDerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (derivationIsImpure(derivationType)) { + if (derivationType.isImpure()) { // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the @@ -1918,7 +1918,7 @@ void LocalDerivationGoal::runChild() sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - if (derivationIsImpure(derivationType)) + if (derivationType.isImpure()) sandboxProfile += "(import \"sandbox-network.sb\")\n"; /* Add the output paths we'll use at build-time to the chroot */ diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 9f21ecf36..de69b50ee 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -560,6 +560,8 @@ static void performOp(TunnelLogger * logger, ref store, BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); + auto drvType = drv.type(); + /* Content-addressed derivations are trustless because their output paths are verified by their content alone, so any derivation is free to try to produce such a path. @@ -592,12 +594,12 @@ static void performOp(TunnelLogger * logger, ref store, derivations, we throw out the precomputed output paths and just store the hashes, so there aren't two competing sources of truth an attacker could exploit. */ - if (drv.type() == DerivationType::InputAddressed && !trusted) + if (!(drvType.isCA() || trusted)) throw Error("you are not privileged to build input-addressed derivations"); /* Make sure that the non-input-addressed derivations that got this far are in fact content-addressed if we don't trust them. */ - assert(derivationIsCA(drv.type()) || trusted); + assert(drvType.isCA() || trusted); /* Recompute the derivation path when we cannot trust the original. */ if (!trusted) { @@ -606,7 +608,7 @@ static void performOp(TunnelLogger * logger, ref store, original not-necessarily-resolved derivation to verify the drv derivation as adequate claim to the input-addressed output paths. */ - assert(derivationIsCA(drv.type())); + assert(drvType.isCA()); Derivation drv2; static_cast(drv2) = drv; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 3dd3a78d8..7fed80387 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,47 +36,46 @@ StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view } -bool derivationIsCA(DerivationType dt) { - switch (dt) { - case DerivationType::InputAddressed: return false; - case DerivationType::CAFixed: return true; - case DerivationType::CAFloating: return true; - case DerivationType::DeferredInputAddressed: return false; - }; - // Since enums can have non-variant values, but making a `default:` would - // disable exhaustiveness warnings. - assert(false); +bool DerivationType::isCA() const { + /* Normally we do the full `std::visit` to make sure we have + exhaustively handled all variants, but so long as there is a + variant called `ContentAddressed`, it must be the only one for + which `isCA` is true for this to make sense!. */ + return std::holds_alternative(raw()); } -bool derivationIsFixed(DerivationType dt) { - switch (dt) { - case DerivationType::InputAddressed: return false; - case DerivationType::CAFixed: return true; - case DerivationType::CAFloating: return false; - case DerivationType::DeferredInputAddressed: return false; - }; - assert(false); +bool DerivationType::isFixed() const { + return std::visit(overloaded { + [](const InputAddressed & ia) { + return false; + }, + [](const ContentAddressed & ca) { + return ca.fixed; + }, + }, raw()); } -bool derivationHasKnownOutputPaths(DerivationType dt) { - switch (dt) { - case DerivationType::InputAddressed: return true; - case DerivationType::CAFixed: return true; - case DerivationType::CAFloating: return false; - case DerivationType::DeferredInputAddressed: return false; - }; - assert(false); +bool DerivationType::hasKnownOutputPaths() const { + return std::visit(overloaded { + [](const InputAddressed & ia) { + return !ia.deferred; + }, + [](const ContentAddressed & ca) { + return ca.fixed; + }, + }, raw()); } -bool derivationIsImpure(DerivationType dt) { - switch (dt) { - case DerivationType::InputAddressed: return false; - case DerivationType::CAFixed: return true; - case DerivationType::CAFloating: return false; - case DerivationType::DeferredInputAddressed: return false; - }; - assert(false); +bool DerivationType::isImpure() const { + return std::visit(overloaded { + [](const InputAddressed & ia) { + return false; + }, + [](const ContentAddressed & ca) { + return !ca.pure; + }, + }, raw()); } @@ -439,18 +438,28 @@ DerivationType BasicDerivation::type() const if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { throw Error("Must have at least one output"); } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { - return DerivationType::InputAddressed; + return DerivationType::InputAddressed { + .deferred = false, + }; } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? throw Error("Only one fixed output is allowed for now"); if (*fixedCAOutputs.begin() != "out") throw Error("Single fixed output must be named \"out\""); - return DerivationType::CAFixed; + return DerivationType::ContentAddressed { + .pure = false, + .fixed = true, + }; } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) { - return DerivationType::CAFloating; + return DerivationType::ContentAddressed { + .pure = true, + .fixed = false, + }; } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) { - return DerivationType::DeferredInputAddressed; + return DerivationType::InputAddressed { + .deferred = true, + }; } else { throw Error("Can't mix derivation output types"); } @@ -502,10 +511,10 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & */ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - auto kind = DrvHash::Kind::Regular; + auto type = drv.type(); + /* Return a fixed hash for fixed-output derivations. */ - switch (drv.type()) { - case DerivationType::CAFixed: { + if (type.isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw()); @@ -517,14 +526,19 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } return outputHashes; } - case DerivationType::CAFloating: - kind = DrvHash::Kind::Deferred; - break; - case DerivationType::InputAddressed: - break; - case DerivationType::DeferredInputAddressed: - break; - } + + auto kind = std::visit(overloaded { + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return DrvHash::Kind::Regular; + }, + [](const DerivationType::ContentAddressed & ca) { + return ca.fixed + ? DrvHash::Kind::Regular + : DrvHash::Kind::Deferred; + }, + }, drv.type().raw()); /* For other derivations, replace the inputs paths with recursive calls to this function. */ diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 6b92b2f21..8dea90abf 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -85,30 +85,50 @@ typedef std::map DerivationInputs; -enum struct DerivationType : uint8_t { - InputAddressed, - DeferredInputAddressed, - CAFixed, - CAFloating, +struct DerivationType_InputAddressed { + bool deferred; }; -/* Do the outputs of the derivation have paths calculated from their content, - or from the derivation itself? */ -bool derivationIsCA(DerivationType); +struct DerivationType_ContentAddressed { + bool pure; + bool fixed; +}; -/* Is the content of the outputs fixed a-priori via a hash? Never true for - non-CA derivations. */ -bool derivationIsFixed(DerivationType); +typedef std::variant< + DerivationType_InputAddressed, + DerivationType_ContentAddressed +> _DerivationTypeRaw; -/* Is the derivation impure and needs to access non-deterministic resources, or - pure and can be sandboxed? Note that whether or not we actually sandbox the - derivation is controlled separately. Never true for non-CA derivations. */ -bool derivationIsImpure(DerivationType); +struct DerivationType : _DerivationTypeRaw { + using Raw = _DerivationTypeRaw; + using Raw::Raw; + using InputAddressed = DerivationType_InputAddressed; + using ContentAddressed = DerivationType_ContentAddressed; -/* Does the derivation knows its own output paths? - * Only true when there's no floating-ca derivation involved in the closure. - */ -bool derivationHasKnownOutputPaths(DerivationType); + + /* Do the outputs of the derivation have paths calculated from their content, + or from the derivation itself? */ + bool isCA() const; + + /* Is the content of the outputs fixed a-priori via a hash? Never true for + non-CA derivations. */ + bool isFixed() const; + + /* Is the derivation impure and needs to access non-deterministic resources, or + pure and can be sandboxed? Note that whether or not we actually sandbox the + derivation is controlled separately. Never true for non-CA derivations. */ + bool isImpure() const; + + /* Does the derivation knows its own output paths? + Only true when there's no floating-ca derivation involved in the + closure, or if fixed output. + */ + bool hasKnownOutputPaths() const; + + inline const Raw & raw() const { + return static_cast(*this); + } +}; struct BasicDerivation { @@ -189,11 +209,11 @@ typedef std::map CaOutputHashes; struct DrvHash { Hash hash; - enum struct Kind { + enum struct Kind: bool { // Statically determined derivations. // This hash will be directly used to compute the output paths Regular, - // Floating-output derivations (and their dependencies). + // Floating-output derivations (and their reverse dependencies). Deferred, }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 75726a1ab..46a547db1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -718,6 +718,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat /* Nothing to check */ }, [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ }, }, i.second.raw()); } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 8c65053e4..f2288a04e 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -93,7 +93,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const StringSet res; for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i); - if (!derivationHasKnownOutputPaths(drv.type())) + if (!drv.type().hasKnownOutputPaths()) res.insert("ca-derivations"); return res; }