From 33656097e48c10768585952efbcb2cb48cf74aa7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 14 Nov 2022 16:26:20 +0100 Subject: [PATCH 1/2] derivation-goal: Fix `requires non-existing output` error It occurred when a output of the dependency was already available, so it didn't need rebuilding and didn't get added to the inputDrvOutputs. This process-related info wasn't suitable for the purpose of finding the actual input paths for the builder. It is better to do this in absolute terms by querying the store. (cherry picked from commit 7e162c69fe6cbfb929b5356a7df9de5c25c22565) --- src/libstore/build/derivation-goal.cc | 30 +++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 83da657f0..aa2b8762f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -528,13 +528,31 @@ void DerivationGoal::inputsRealised() /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - for (auto & j : wantedDepOutputs) - if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) + for (auto & j : wantedDepOutputs) { + /* TODO (impure derivations-induced tech debt): + Tracking input derivation outputs statefully through the + goals is error prone and has led to bugs. + For a robust nix, we need to move towards the `else` branch, + which does not rely on goal state to match up with the + reality of the store, which is our real source of truth. + However, the impure derivations feature still relies on this + fragile way of doing things, because its builds do not have + a representation in the store, which is a usability problem + in itself */ + if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { worker.store.computeFSClosure(*outPath, inputPaths); - else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + else { + auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); + auto outMapPath = outMap.find(j); + if (outMapPath == outMap.end()) { + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + worker.store.computeFSClosure(outMapPath->second, inputPaths); + } + } } } From a5199bd82689aa7afd14c9caff286366a9b6d74e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 9 Nov 2022 15:21:13 +0100 Subject: [PATCH 2/2] tests: Reproduce #6572 (cherry picked from commit c279ddb18cf3a34b0f6d4e3adcf9455da5397ad7) --- tests/build.sh | 51 ++++++++++++++++++++++++++++++++++++++ tests/multiple-outputs.nix | 30 ++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/tests/build.sh b/tests/build.sh index fc6825e25..c7db039b4 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -70,3 +70,54 @@ testNormalization () { } testNormalization + +# https://github.com/NixOS/nix/issues/6572 +issue_6572_independent_outputs() { + nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json + + # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. + p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) + nix-store --delete "$p" # Clean up for next test + + # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. + nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)" + p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) + cmp $p < $TEST_ROOT/a.json + + # # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. + p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) + nix-store --delete "$p" # Clean up for next test + + # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. + nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)" + p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) + cmp $p <$out + ''; + }; + b = mkDerivation { defaultOutput = assert a.second.helloString == "Hello, world!"; a; firstOutput = assert a.outputName == "first"; a.first.first; @@ -87,4 +96,25 @@ rec { buildCommand = "mkdir $a $b $c"; }; + independent = mkDerivation { + name = "multiple-outputs-independent"; + outputs = [ "first" "second" ]; + builder = builtins.toFile "builder.sh" + '' + mkdir $first $second + test -z $all + echo "first" > $first/file + echo "second" > $second/file + ''; + }; + + use-independent = mkDerivation { + name = "use-independent"; + inherit (a) first second; + builder = builtins.toFile "builder.sh" + '' + cat $first/file $second/file >$out + ''; + }; + }