nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-14 04:51:05 +01:00
Code Activity

Split out DerivationResolutionGoal

Browse source 
This prepares the way for fixing a few issues.
This commit is contained in:
John Ericson 2025-09-18 15:54:43 -04:00
parent d76dc2406f
commit 8f4a739d0f
9 changed files with 334 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

135
src/libstore/build/derivation-building-goal.cc
View file

@ -1,4 +1,5 @@
#include "nix/store/build/derivation-building-goal.hh"
#include "nix/store/build/derivation-resolution-goal.hh"
#include "nix/store/build/derivation-env-desugar.hh"
#include "nix/store/build/derivation-trampoline-goal.hh"
#ifndef _WIN32 // TODO enable build hook on Windows
@ -129,46 +130,6 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
{
Goals waitees;
std::map<ref<const SingleDerivedPath>, GoalPtr, value_comparison> inputGoals;
{
std::function<void(ref<const SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)>
addWaiteeDerivedPath;
addWaiteeDerivedPath = [&](ref<const SingleDerivedPath> inputDrv,
const DerivedPathMap<StringSet>::ChildNode & inputNode) {
if (!inputNode.value.empty()) {
auto g = worker.makeGoal(
DerivedPath::Built{
.drvPath = inputDrv,
.outputs = inputNode.value,
},
buildMode == bmRepair ? bmRepair : bmNormal);
inputGoals.insert_or_assign(inputDrv, g);
waitees.insert(std::move(g));
}
for (const auto & [outputName, childNode] : inputNode.childMap)
addWaiteeDerivedPath(
make_ref<SingleDerivedPath>(SingleDerivedPath::Built{inputDrv, outputName}), childNode);
};
for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) {
/* Ensure that pure, non-fixed-output derivations don't
depend on impure derivations. */
if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure()
&& !drv->type().isFixed()) {
auto inputDrv = worker.evalStore.readDerivation(inputDrvPath);
if (inputDrv.type().isImpure())
throw Error(
"pure derivation '%s' depends on impure derivation '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(inputDrvPath));
}
addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode);
}
}
/* Copy the input sources from the eval store to the build
store.
@ -213,88 +174,22 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
/* Determine the full set of input paths. */
/* First, the input derivations. */
{
auto & fullDrv = *drv;
auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, *drv, buildMode);
{
Goals waitees{resolutionGoal};
co_await await(std::move(waitees));
}
if (nrFailed != 0) {
co_return doneFailure({BuildResult::Failure::DependencyFailed, "resolution failed"});
}
auto drvType = fullDrv.type();
bool resolveDrv =
std::visit(
overloaded{
[&](const DerivationType::InputAddressed & ia) {
/* must resolve if deferred. */
return ia.deferred;
},
[&](const DerivationType::ContentAddressed & ca) {
return !fullDrv.inputDrvs.map.empty()
&& (ca.fixed
/* Can optionally resolve if fixed, which is good
for avoiding unnecessary rebuilds. */
? experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
/* Must resolve if floating and there are any inputs
drvs. */
: true);
},
[&](const DerivationType::Impure &) { return true; }},
drvType.raw)
/* no inputs are outputs of dynamic derivations */
|| std::ranges::any_of(fullDrv.inputDrvs.map.begin(), fullDrv.inputDrvs.map.end(), [](auto & pair) {
return !pair.second.childMap.empty();
});
if (resolutionGoal->resolvedDrv) {
auto & [pathResolved, drvResolved] = *resolutionGoal->resolvedDrv;
if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
experimentalFeatureSettings.require(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a
stub goal aliasing that resolved derivation goal. */
std::optional attempt = fullDrv.tryResolve(
worker.store,
[&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
auto mEntry = get(inputGoals, drvPath);
if (!mEntry)
return std::nullopt;
auto & buildResult = (*mEntry)->buildResult;
return std::visit(
overloaded{
[](const BuildResult::Failure &) -> std::optional<StorePath> { return std::nullopt; },
[&](const BuildResult::Success & success) -> std::optional<StorePath> {
auto i = get(success.builtOutputs, outputName);
if (!i)
return std::nullopt;
return i->outPath;
},
},
buildResult.inner);
});
if (!attempt) {
/* TODO (impure derivations-induced tech debt) (see below):
The above attempt should have found it, but because we manage
inputDrvOutputs statefully, sometimes it gets out of sync with
the real source of truth (store). So we query the store
directly if there's a problem. */
attempt = fullDrv.tryResolve(worker.store, &worker.evalStore);
}
assert(attempt);
Derivation drvResolved{std::move(*attempt)};
auto pathResolved = writeDerivation(worker.store, drvResolved);
auto msg =
fmt("resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved));
act = std::make_unique<Activity>(
*logger,
lvlInfo,
actBuildWaiting,
msg,
Logger::Fields{
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved),
});
/* Store the resolved derivation, as part of the record of
what we're actually building */
writeDerivation(worker.store, drvResolved);
/* TODO https://github.com/NixOS/nix/issues/13247 we should
let the calling goal do this, so it has a change to pass
@ -383,7 +278,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
/* If we get this far, we know no dynamic drvs inputs */
for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) {
for (auto & [depDrvPath, depNode] : drv->inputDrvs.map) {
for (auto & outputName : depNode.value) {
/* Don't need to worry about `inputGoals`, because
impure derivations are always resolved above. Can

210
src/libstore/build/derivation-resolution-goal.cc Normal file
View file

@ -0,0 +1,210 @@
#include "nix/store/build/derivation-resolution-goal.hh"
#include "nix/store/build/derivation-env-desugar.hh"
#include "nix/store/build/worker.hh"
#include "nix/util/util.hh"
#include "nix/store/common-protocol.hh"
#include "nix/store/globals.hh"
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
#include <nlohmann/json.hpp>
namespace nix {
DerivationResolutionGoal::DerivationResolutionGoal(
const StorePath & drvPath, const Derivation & drv_, Worker & worker, BuildMode buildMode)
: Goal(worker, resolveDerivation())
, drvPath(drvPath)
{
drv = std::make_unique<Derivation>(drv_);
name = fmt("building of '%s' from in-memory derivation", worker.store.printStorePath(drvPath));
trace("created");
/* Prevent the .chroot directory from being
garbage-collected. (See isActiveTempFile() in gc.cc.) */
worker.store.addTempRoot(this->drvPath);
}
void DerivationResolutionGoal::timedOut(Error && ex) {}
std::string DerivationResolutionGoal::key()
{
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before
"baboon". And substitution goals always happen before
derivation goals (due to "bd$"). */
return "rd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
}
/**
* Used for `inputGoals` local variable below
*/
struct value_comparison
{
template<typename T>
bool operator()(const ref<T> & lhs, const ref<T> & rhs) const
{
return *lhs < *rhs;
}
};
/* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */
Goal::Co DerivationResolutionGoal::resolveDerivation()
{
Goals waitees;
std::map<ref<const SingleDerivedPath>, GoalPtr, value_comparison> inputGoals;
{
std::function<void(ref<const SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)>
addWaiteeDerivedPath;
addWaiteeDerivedPath = [&](ref<const SingleDerivedPath> inputDrv,
const DerivedPathMap<StringSet>::ChildNode & inputNode) {
if (!inputNode.value.empty()) {
auto g = worker.makeGoal(
DerivedPath::Built{
.drvPath = inputDrv,
.outputs = inputNode.value,
},
buildMode == bmRepair ? bmRepair : bmNormal);
inputGoals.insert_or_assign(inputDrv, g);
waitees.insert(std::move(g));
}
for (const auto & [outputName, childNode] : inputNode.childMap)
addWaiteeDerivedPath(
make_ref<SingleDerivedPath>(SingleDerivedPath::Built{inputDrv, outputName}), childNode);
};
for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) {
/* Ensure that pure, non-fixed-output derivations don't
depend on impure derivations. */
if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure()
&& !drv->type().isFixed()) {
auto inputDrv = worker.evalStore.readDerivation(inputDrvPath);
if (inputDrv.type().isImpure())
throw Error(
"pure derivation '%s' depends on impure derivation '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(inputDrvPath));
}
addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode);
}
}
co_await await(std::move(waitees));
trace("all inputs realised");
if (nrFailed != 0) {
auto msg =
fmt("Cannot build '%s'.\n"
"Reason: " ANSI_RED "%d %s failed" ANSI_NORMAL ".",
Magenta(worker.store.printStorePath(drvPath)),
nrFailed,
nrFailed == 1 ? "dependency" : "dependencies");
msg += showKnownOutputs(worker.store, *drv);
co_return amDone(ecFailed, {BuildError(BuildResult::Failure::DependencyFailed, msg)});
}
/* Gather information necessary for computing the closure and/or
running the build hook. */
/* Determine the full set of input paths. */
/* First, the input derivations. */
{
auto & fullDrv = *drv;
auto drvType = fullDrv.type();
bool resolveDrv =
std::visit(
overloaded{
[&](const DerivationType::InputAddressed & ia) {
/* must resolve if deferred. */
return ia.deferred;
},
[&](const DerivationType::ContentAddressed & ca) {
return !fullDrv.inputDrvs.map.empty()
&& (ca.fixed
/* Can optionally resolve if fixed, which is good
for avoiding unnecessary rebuilds. */
? experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
/* Must resolve if floating and there are any inputs
drvs. */
: true);
},
[&](const DerivationType::Impure &) { return true; }},
drvType.raw)
/* no inputs are outputs of dynamic derivations */
|| std::ranges::any_of(fullDrv.inputDrvs.map.begin(), fullDrv.inputDrvs.map.end(), [](auto & pair) {
return !pair.second.childMap.empty();
});
if (resolveDrv && !fullDrv.inputDrvs.map.empty()) {
experimentalFeatureSettings.require(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a
stub goal aliasing that resolved derivation goal. */
std::optional attempt = fullDrv.tryResolve(
worker.store,
[&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
auto mEntry = get(inputGoals, drvPath);
if (!mEntry)
return std::nullopt;
auto & buildResult = (*mEntry)->buildResult;
return std::visit(
overloaded{
[](const BuildResult::Failure &) -> std::optional<StorePath> { return std::nullopt; },
[&](const BuildResult::Success & success) -> std::optional<StorePath> {
auto i = get(success.builtOutputs, outputName);
if (!i)
return std::nullopt;
return i->outPath;
},
},
buildResult.inner);
});
if (!attempt) {
/* TODO (impure derivations-induced tech debt) (see below):
The above attempt should have found it, but because we manage
inputDrvOutputs statefully, sometimes it gets out of sync with
the real source of truth (store). So we query the store
directly if there's a problem. */
attempt = fullDrv.tryResolve(worker.store, &worker.evalStore);
}
assert(attempt);
auto pathResolved = writeDerivation(worker.store, *attempt, NoRepair, /*readOnly =*/true);
auto msg =
fmt("resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved));
act = std::make_unique<Activity>(
*logger,
lvlInfo,
actBuildWaiting,
msg,
Logger::Fields{
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved),
});
resolvedDrv =
std::make_unique<std::pair<StorePath, BasicDerivation>>(std::move(pathResolved), *std::move(attempt));
}
}
co_return amDone(ecSuccess, std::nullopt);
}
} // namespace nix

9
src/libstore/build/worker.cc
View file

@ -4,6 +4,7 @@
#include "nix/store/build/substitution-goal.hh"
#include "nix/store/build/drv-output-substitution-goal.hh"
#include "nix/store/build/derivation-goal.hh"
#include "nix/store/build/derivation-resolution-goal.hh"
#include "nix/store/build/derivation-building-goal.hh"
#include "nix/store/build/derivation-trampoline-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows
@ -80,6 +81,12 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(
return initGoalIfNeeded(derivationGoals[drvPath][wantedOutput], drvPath, drv, wantedOutput, *this, buildMode);
}
std::shared_ptr<DerivationResolutionGoal>
Worker::makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode)
{
return initGoalIfNeeded(derivationResolutionGoals[drvPath], drvPath, drv, *this, buildMode);
}
std::shared_ptr<DerivationBuildingGoal>
Worker::makeDerivationBuildingGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode)
{
@ -158,6 +165,8 @@ void Worker::removeGoal(GoalPtr goal)
nix::removeGoal(drvGoal, derivationTrampolineGoals.map);
else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else if (auto drvResolutionGoal = std::dynamic_pointer_cast<DerivationResolutionGoal>(goal))
nix::removeGoal(drvResolutionGoal, derivationResolutionGoals);
else if (auto drvBuildingGoal = std::dynamic_pointer_cast<DerivationBuildingGoal>(goal))
nix::removeGoal(drvBuildingGoal, derivationBuildingGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))