nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-11 03:21:03 +01:00
Code Activity

Merge pull request #13240 from obsidiansystems/dyn-drv-take-3

Browse source 
Get rid of `addWantedOutputs`
This commit is contained in:
John Ericson 2025-07-07 11:39:45 -04:00 • committed by GitHub
commit cf0df2607d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 481 additions and 361 deletions
Download patch file Download diff file Expand all files Collapse all files

26
src/libstore/build/derivation-building-goal.cc
View file

@ -1,5 +1,5 @@
#include "nix/store/build/derivation-building-goal.hh"
#include "nix/store/build/derivation-goal.hh"
#include "nix/store/build/derivation-trampoline-goal.hh"
#ifndef _WIN32 // TODO enable build hook on Windows
# include "nix/store/build/hook-instance.hh"
# include "nix/store/build/derivation-builder.hh"
@ -72,7 +72,7 @@ std::string DerivationBuildingGoal::key()
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before
"baboon". And substitution goals always happen before
derivation goals (due to "b$"). */
derivation goals (due to "bd$"). */
return "bd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
}
@ -266,7 +266,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
auto mEntry = get(inputGoals, drvPath);
if (!mEntry) return std::nullopt;
auto buildResult = (*mEntry)->getBuildResult(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}});
auto & buildResult = (*mEntry)->buildResult;
if (!buildResult.success()) return std::nullopt;
auto i = get(buildResult.builtOutputs, outputName);
@ -296,9 +296,11 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
worker.store.printStorePath(pathResolved),
});
// FIXME wanted outputs
auto resolvedDrvGoal = worker.makeDerivationGoal(
makeConstantStorePathRef(pathResolved), OutputsSpec::All{}, buildMode);
/* TODO https://github.com/NixOS/nix/issues/13247 we should
let the calling goal do this, so it has a change to pass
just the output(s) it cares about. */
auto resolvedDrvGoal = worker.makeDerivationTrampolineGoal(
pathResolved, OutputsSpec::All{}, drvResolved, buildMode);
{
Goals waitees{resolvedDrvGoal};
co_await await(std::move(waitees));
@ -306,20 +308,16 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
trace("resolved derivation finished");
auto resolvedDrv = *resolvedDrvGoal->drv;
auto resolvedResult = resolvedDrvGoal->getBuildResult(DerivedPath::Built{
.drvPath = makeConstantStorePathRef(pathResolved),
.outputs = OutputsSpec::All{},
});
auto resolvedResult = resolvedDrvGoal->buildResult;
SingleDrvOutputs builtOutputs;
if (resolvedResult.success()) {
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
auto resolvedHashes = staticOutputHashes(worker.store, drvResolved);
StorePathSet outputPaths;
for (auto & outputName : resolvedDrv.outputNames()) {
for (auto & outputName : drvResolved.outputNames()) {
auto initialOutput = get(initialOutputs, outputName);
auto resolvedHash = get(resolvedHashes, outputName);
if ((!initialOutput) || (!resolvedHash))
@ -340,7 +338,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/realisation)",
resolvedDrvGoal->drvReq->to_string(worker.store), outputName);
worker.store.printStorePath(pathResolved), outputName);
}();
if (!drv->type().isImpure()) {

205
src/libstore/build/derivation-goal.cc
View file

@ -24,35 +24,18 @@
namespace nix {
DerivationGoal::DerivationGoal(ref<const SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, loadDerivation())
, drvReq(drvReq)
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
name = fmt(
"building of '%s' from .drv file",
DerivedPath::Built { drvReq, wantedOutputs }.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
worker.updateProgress();
}
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, haveDerivation(drvPath))
, drvReq(makeConstantStorePathRef(drvPath))
, wantedOutputs(wantedOutputs)
DerivationGoal::DerivationGoal(const StorePath & drvPath, const Derivation & drv,
const OutputName & wantedOutput, Worker & worker, BuildMode buildMode)
: Goal(worker, haveDerivation())
, drvPath(drvPath)
, wantedOutput(wantedOutput)
, buildMode(buildMode)
{
this->drv = std::make_unique<Derivation>(drv);
name = fmt(
"building of '%s' from in-memory derivation",
DerivedPath::Built { drvReq, drv.outputNames() }.to_string(worker.store));
DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@ -61,114 +44,20 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
}
static StorePath pathPartOfReq(const SingleDerivedPath & req)
{
return std::visit(
overloaded{
[&](const SingleDerivedPath::Opaque & bo) { return bo.path; },
[&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); },
},
req.raw());
}
std::string DerivationGoal::key()
{
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before
"baboon". And substitution goals always happen before
derivation goals (due to "b$"). */
return "b$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
return "b$" + std::string(drvPath.name()) + "$" + SingleDerivedPath::Built{
.drvPath = makeConstantStorePathRef(drvPath),
.output = wantedOutput,
}.to_string(worker.store);
}
void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
{
auto newWanted = wantedOutputs.union_(outputs);
switch (needRestart) {
case NeedRestartForMoreOutputs::OutputsUnmodifiedDontNeed:
if (!newWanted.isSubsetOf(wantedOutputs))
needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed;
break;
case NeedRestartForMoreOutputs::OutputsAddedDoNeed:
/* No need to check whether we added more outputs, because a
restart is already queued up. */
break;
case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed:
/* We are already building all outputs, so it doesn't matter if
we now want more. */
break;
};
wantedOutputs = newWanted;
}
Goal::Co DerivationGoal::loadDerivation() {
trace("need to load derivation from file");
{
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be built from another
derivation, or merely substituted. We can make goal to get it
and not worry about which method it takes to get the
derivation. */
if (auto optDrvPath = [this]() -> std::optional<StorePath> {
if (buildMode != bmNormal)
return std::nullopt;
auto drvPath = StorePath::dummy;
try {
drvPath = resolveDerivedPath(worker.store, *drvReq);
} catch (MissingRealisation &) {
return std::nullopt;
}
auto cond = worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath);
return cond ? std::optional{drvPath} : std::nullopt;
}()) {
trace(
fmt("already have drv '%s' for '%s', can go straight to building",
worker.store.printStorePath(*optDrvPath),
drvReq->to_string(worker.store)));
} else {
trace("need to obtain drv we want to build");
Goals waitees{worker.makeGoal(DerivedPath::fromSingle(*drvReq))};
co_await await(std::move(waitees));
}
trace("loading derivation");
if (nrFailed != 0) {
co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
}
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
/* `drvPath' should already be a root, but let's be on the safe
side: if the user forgot to make it a root, we wouldn't want
things being garbage collected while we're busy. */
worker.evalStore.addTempRoot(drvPath);
/* Get the derivation. It is probably in the eval store, but it might be inthe main store:
- Resolved derivation are resolved against main store realisations, and so must be stored there.
- Dynamic derivations are built, and so are found in the main store.
*/
for (auto * drvStore : { &worker.evalStore, &worker.store }) {
if (drvStore->isValidPath(drvPath)) {
drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
break;
}
}
assert(drv);
co_return haveDerivation(drvPath);
}
}
Goal::Co DerivationGoal::haveDerivation(StorePath drvPath)
Goal::Co DerivationGoal::haveDerivation()
{
trace("have derivation");
@ -205,18 +94,26 @@ Goal::Co DerivationGoal::haveDerivation(StorePath drvPath)
trace("outer build done");
buildResult = g->getBuildResult(DerivedPath::Built{
.drvPath = makeConstantStorePathRef(drvPath),
.outputs = wantedOutputs,
});
buildResult = g->buildResult;
if (buildMode == bmCheck) {
/* In checking mode, the builder will not register any outputs.
So we want to make sure the ones that we wanted to check are
properly there. */
buildResult.builtOutputs = assertPathValidity(drvPath);
buildResult.builtOutputs = assertPathValidity();
}
for (auto it = buildResult.builtOutputs.begin(); it != buildResult.builtOutputs.end(); ) {
if (it->first != wantedOutput) {
it = buildResult.builtOutputs.erase(it);
} else {
++it;
}
}
if (buildResult.success())
assert(buildResult.builtOutputs.count(wantedOutput) > 0);
co_return amDone(g->exitCode, g->ex);
};
@ -261,11 +158,11 @@ Goal::Co DerivationGoal::haveDerivation(StorePath drvPath)
{
/* Check what outputs paths are not already valid. */
auto [allValid, validOutputs] = checkPathValidity(drvPath);
auto [allValid, validOutputs] = checkPathValidity();
/* If they are all valid, then we're done. */
if (allValid && buildMode == bmNormal) {
co_return done(drvPath, BuildResult::AlreadyValid, std::move(validOutputs));
co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
}
}
@ -302,25 +199,20 @@ Goal::Co DerivationGoal::haveDerivation(StorePath drvPath)
assert(!drv->type().isImpure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) {
co_return done(drvPath, BuildResult::TransientFailure, {},
co_return done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
worker.store.printStorePath(drvPath)));
}
nrFailed = nrNoSubstituters = 0;
if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) {
needRestart = NeedRestartForMoreOutputs::OutputsUnmodifiedDontNeed;
co_return haveDerivation(std::move(drvPath));
}
auto [allValid, validOutputs] = checkPathValidity(drvPath);
auto [allValid, validOutputs] = checkPathValidity();
if (buildMode == bmNormal && allValid) {
co_return done(drvPath, BuildResult::Substituted, std::move(validOutputs));
co_return done(BuildResult::Substituted, std::move(validOutputs));
}
if (buildMode == bmRepair && allValid) {
co_return repairClosure(std::move(drvPath));
co_return repairClosure();
}
if (buildMode == bmCheck && !allValid)
throw Error("some outputs of '%s' are not valid, so checking is not possible",
@ -343,7 +235,7 @@ struct value_comparison
};
Goal::Co DerivationGoal::repairClosure(StorePath drvPath)
Goal::Co DerivationGoal::repairClosure()
{
assert(!drv->type().isImpure());
@ -353,11 +245,10 @@ Goal::Co DerivationGoal::repairClosure(StorePath drvPath)
that produced those outputs. */
/* Get the output closure. */
auto outputs = queryDerivationOutputMap(drvPath);
auto outputs = queryDerivationOutputMap();
StorePathSet outputClosure;
for (auto & i : outputs) {
if (!wantedOutputs.contains(i.first)) continue;
worker.store.computeFSClosure(i.second, outputClosure);
if (auto mPath = get(outputs, wantedOutput)) {
worker.store.computeFSClosure(*mPath, outputClosure);
}
/* Filter out our own outputs (which we have already checked). */
@ -411,11 +302,11 @@ Goal::Co DerivationGoal::repairClosure(StorePath drvPath)
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
worker.store.printStorePath(drvPath));
}
co_return done(drvPath, BuildResult::AlreadyValid, assertPathValidity(drvPath));
co_return done(BuildResult::AlreadyValid, assertPathValidity());
}
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap(const StorePath & drvPath)
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{
assert(!drv->type().isImpure());
@ -431,7 +322,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
return res;
}
OutputPathMap DerivationGoal::queryDerivationOutputMap(const StorePath & drvPath)
OutputPathMap DerivationGoal::queryDerivationOutputMap()
{
assert(!drv->type().isImpure());
@ -447,28 +338,21 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap(const StorePath & drvPath
}
std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity(const StorePath & drvPath)
std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
{
if (drv->type().isImpure()) return { false, {} };
bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = std::visit(overloaded {
[&](const OutputsSpec::All &) {
return StringSet {};
},
[&](const OutputsSpec::Names & names) {
return static_cast<StringSet>(names);
},
}, wantedOutputs.raw);
StringSet wantedOutputsLeft{wantedOutput};
SingleDrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap(drvPath)) {
for (auto & i : queryPartialDerivationOutputMap()) {
auto initialOutput = get(initialOutputs, i.first);
if (!initialOutput)
// this is an invalid output, gets caught with (!wantedOutputsLeft.empty())
continue;
auto & info = *initialOutput;
info.wanted = wantedOutputs.contains(i.first);
info.wanted = wantedOutput == i.first;
if (info.wanted)
wantedOutputsLeft.erase(i.first);
if (i.second) {
@ -527,9 +411,9 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity(const StoreP
}
SingleDrvOutputs DerivationGoal::assertPathValidity(const StorePath & drvPath)
SingleDrvOutputs DerivationGoal::assertPathValidity()
{
auto [allValid, validOutputs] = checkPathValidity(drvPath);
auto [allValid, validOutputs] = checkPathValidity();
if (!allValid)
throw Error("some outputs are unexpectedly invalid");
return validOutputs;
@ -537,7 +421,6 @@ SingleDrvOutputs DerivationGoal::assertPathValidity(const StorePath & drvPath)
Goal::Done DerivationGoal::done(
const StorePath & drvPath,
BuildResult::Status status,
SingleDrvOutputs builtOutputs,
std::optional<Error> ex)
@ -553,7 +436,7 @@ Goal::Done DerivationGoal::done(
mcExpectedBuilds.reset();
if (buildResult.success()) {
auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs));
auto wantedBuiltOutputs = filterDrvOutputs(OutputsSpec::Names{wantedOutput}, std::move(builtOutputs));
assert(!wantedBuiltOutputs.empty());
buildResult.builtOutputs = std::move(wantedBuiltOutputs);
if (status == BuildResult::Built)

175
src/libstore/build/derivation-trampoline-goal.cc Normal file
View file

@ -0,0 +1,175 @@
#include "nix/store/build/derivation-trampoline-goal.hh"
#include "nix/store/build/worker.hh"
#include "nix/store/derivations.hh"
namespace nix {
DerivationTrampolineGoal::DerivationTrampolineGoal(
ref<const SingleDerivedPath> drvReq, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, init())
, drvReq(drvReq)
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
commonInit();
}
DerivationTrampolineGoal::DerivationTrampolineGoal(
const StorePath & drvPath,
const OutputsSpec & wantedOutputs,
const Derivation & drv,
Worker & worker,
BuildMode buildMode)
: Goal(worker, haveDerivation(drvPath, drv))
, drvReq(makeConstantStorePathRef(drvPath))
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
commonInit();
}
void DerivationTrampolineGoal::commonInit()
{
name =
fmt("outer obtaining drv from '%s' and then building outputs %s",
drvReq->to_string(worker.store),
std::visit(
overloaded{
[&](const OutputsSpec::All) -> std::string { return "* (all of them)"; },
[&](const OutputsSpec::Names os) { return concatStringsSep(", ", quoteStrings(os)); },
},
wantedOutputs.raw));
trace("created outer");
worker.updateProgress();
}
DerivationTrampolineGoal::~DerivationTrampolineGoal() {}
static StorePath pathPartOfReq(const SingleDerivedPath & req)
{
return std::visit(
overloaded{
[&](const SingleDerivedPath::Opaque & bo) { return bo.path; },
[&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); },
},
req.raw());
}
std::string DerivationTrampolineGoal::key()
{
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before "baboon". And
substitution goals, derivation goals, and derivation building goals always happen before
derivation goals (due to "bt$"). */
return "bt$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + DerivedPath::Built{
.drvPath = drvReq,
.outputs = wantedOutputs,
}.to_string(worker.store);
}
void DerivationTrampolineGoal::timedOut(Error && ex) {}
Goal::Co DerivationTrampolineGoal::init()
{
trace("need to load derivation from file");
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be built from another derivation,
or merely substituted. We can make goal to get it and not worry
about which method it takes to get the derivation. */
if (auto optDrvPath = [this]() -> std::optional<StorePath> {
if (buildMode != bmNormal)
return std::nullopt;
auto drvPath = StorePath::dummy;
try {
drvPath = resolveDerivedPath(worker.store, *drvReq);
} catch (MissingRealisation &) {
return std::nullopt;
}
auto cond = worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath);
return cond ? std::optional{drvPath} : std::nullopt;
}()) {
trace(
fmt("already have drv '%s' for '%s', can go straight to building",
worker.store.printStorePath(*optDrvPath),
drvReq->to_string(worker.store)));
} else {
trace("need to obtain drv we want to build");
Goals waitees{worker.makeGoal(DerivedPath::fromSingle(*drvReq))};
co_await await(std::move(waitees));
}
trace("outer load and build derivation");
if (nrFailed != 0) {
co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
}
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
/* `drvPath' should already be a root, but let's be on the safe
side: if the user forgot to make it a root, we wouldn't want
things being garbage collected while we're busy. */
worker.evalStore.addTempRoot(drvPath);
/* Get the derivation. It is probably in the eval store, but it might be in the main store:
- Resolved derivation are resolved against main store realisations, and so must be stored there.
- Dynamic derivations are built, and so are found in the main store.
*/
auto drv = [&] {
for (auto * drvStore : {&worker.evalStore, &worker.store})
if (drvStore->isValidPath(drvPath))
return drvStore->readDerivation(drvPath);
assert(false);
}();
co_return haveDerivation(std::move(drvPath), std::move(drv));
}
Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation drv)
{
trace("have derivation, will kick off derivations goals per wanted output");
auto resolvedWantedOutputs = std::visit(
overloaded{
[&](const OutputsSpec::Names & names) -> OutputsSpec::Names { return names; },
[&](const OutputsSpec::All &) -> OutputsSpec::Names {
StringSet outputs;
for (auto & [outputName, _] : drv.outputs)
outputs.insert(outputName);
return outputs;
},
},
wantedOutputs.raw);
Goals concreteDrvGoals;
/* Build this step! */
for (auto & output : resolvedWantedOutputs) {
auto g = upcast_goal(worker.makeDerivationGoal(drvPath, drv, output, buildMode));
g->preserveException = true;
/* We will finish with it ourselves, as if we were the derivational goal. */
concreteDrvGoals.insert(std::move(g));
}
// Copy on purpose
co_await await(Goals(concreteDrvGoals));
trace("outer build done");
auto & g = *concreteDrvGoals.begin();
buildResult = g->buildResult;
for (auto & g2 : concreteDrvGoals) {
for (auto && [x, y] : g2->buildResult.builtOutputs)
buildResult.builtOutputs.insert_or_assign(x, y);
}
co_return amDone(g->exitCode, g->ex);
}
}

26
src/libstore/build/entry-points.cc
View file

@ -1,8 +1,7 @@
#include "nix/store/derivations.hh"
#include "nix/store/build/worker.hh"
#include "nix/store/build/substitution-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows
# include "nix/store/build/derivation-goal.hh"
#endif
#include "nix/store/build/derivation-trampoline-goal.hh"
#include "nix/store/local-store.hh"
#include "nix/util/strings.hh"
@ -28,12 +27,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
ex = std::move(i->ex);
}
if (i->exitCode != Goal::ecSuccess) {
#ifndef _WIN32 // TODO Enable building on Windows
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
if (auto i2 = dynamic_cast<DerivationTrampolineGoal *>(i.get()))
failed.insert(i2->drvReq->to_string(*this));
else
#endif
if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
failed.insert(printStorePath(i2->storePath));
}
}
@ -70,7 +66,7 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
for (auto & [req, goalPtr] : state)
results.emplace_back(KeyedBuildResult {
goalPtr->getBuildResult(req),
goalPtr->buildResult,
/* .path = */ req,
});
@ -81,19 +77,11 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
BuildMode buildMode)
{
Worker worker(*this, *this);
#ifndef _WIN32 // TODO Enable building on Windows
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode);
#else
std::shared_ptr<Goal> goal;
throw UnimplementedError("Building derivations not yet implemented on windows.");
#endif
auto goal = worker.makeDerivationTrampolineGoal(drvPath, OutputsSpec::All {}, drv, buildMode);
try {
worker.run(Goals{goal});
return goal->getBuildResult(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath),
.outputs = OutputsSpec::All {},
});
return goal->buildResult;
} catch (Error & e) {
return BuildResult {
.status = BuildResult::MiscFailure,

24
src/libstore/build/goal.cc
View file

@ -101,30 +101,6 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
return s1 < s2;
}
BuildResult Goal::getBuildResult(const DerivedPath & req) const {
BuildResult res { buildResult };
if (auto pbp = std::get_if<DerivedPath::Built>(&req)) {
auto & bp = *pbp;
/* Because goals are in general shared between derived paths
that share the same derivation, we need to filter their
results to get back just the results we care about.
*/
for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) {
if (bp.outputs.contains(it->first))
++it;
else
it = res.builtOutputs.erase(it);
}
}
return res;
}
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
{
if (goals.find(p) != goals.end())

122
src/libstore/build/worker.cc
View file

@ -5,6 +5,7 @@
#include "nix/store/build/drv-output-substitution-goal.hh"
#include "nix/store/build/derivation-goal.hh"
#include "nix/store/build/derivation-building-goal.hh"
#include "nix/store/build/derivation-trampoline-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows
# include "nix/store/build/hook-instance.hh"
#endif
@ -53,52 +54,40 @@ std::shared_ptr<G> Worker::initGoalIfNeeded(std::weak_ptr<G> & goal_weak, Args &
return goal;
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
std::shared_ptr<DerivationTrampolineGoal> Worker::makeDerivationTrampolineGoal(
ref<const SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
BuildMode buildMode)
{
std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals.ensureSlot(*drvReq).value;
std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
if (!goal) {
goal = mkDrvGoal();
goal_weak = goal;
wakeUp(goal);
} else {
goal->addWantedOutputs(wantedOutputs);
}
return goal;
return initGoalIfNeeded(
derivationTrampolineGoals.ensureSlot(*drvReq).value[wantedOutputs],
drvReq, wantedOutputs, *this, buildMode);
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(ref<const SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs, BuildMode buildMode)
std::shared_ptr<DerivationTrampolineGoal> Worker::makeDerivationTrampolineGoal(
const StorePath & drvPath,
const OutputsSpec & wantedOutputs,
const Derivation & drv,
BuildMode buildMode)
{
return makeDerivationGoalCommon(drvReq, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return std::make_shared<DerivationGoal>(drvReq, wantedOutputs, *this, buildMode);
});
return initGoalIfNeeded(
derivationTrampolineGoals.ensureSlot(DerivedPath::Opaque{drvPath}).value[wantedOutputs],
drvPath, wantedOutputs, drv, *this, buildMode);
}
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
const Derivation & drv, const OutputName & wantedOutput, BuildMode buildMode)
{
return makeDerivationGoalCommon(makeConstantStorePathRef(drvPath), wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
});
return initGoalIfNeeded(derivationGoals[drvPath][wantedOutput], drvPath, drv, wantedOutput, *this, buildMode);
}
std::shared_ptr<DerivationBuildingGoal> Worker::makeDerivationBuildingGoal(const StorePath & drvPath,
const Derivation & drv, BuildMode buildMode)
{
std::weak_ptr<DerivationBuildingGoal> & goal_weak = derivationBuildingGoals[drvPath];
auto goal = goal_weak.lock(); // FIXME
if (!goal) {
goal = std::make_shared<DerivationBuildingGoal>(drvPath, drv, *this, buildMode);
goal_weak = goal;
wakeUp(goal);
}
return goal;
return initGoalIfNeeded(derivationBuildingGoals[drvPath], drvPath, drv, *this, buildMode);
}
@ -118,7 +107,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr {
return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode);
return makeDerivationTrampolineGoal(bfd.drvPath, bfd.outputs, buildMode);
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@ -126,46 +115,52 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
}, req.raw());
}
template<typename K, typename V, typename F>
static void cullMap(std::map<K, V> & goalMap, F f)
/**
* This function is polymorphic (both via type parameters and
* overloading) and recursive in order to work on a various types of
* trees
*
* @return Whether the tree node we are processing is not empty / should
* be kept alive. In the case of this overloading the node in question
* is the leaf, the weak reference itself. If the weak reference points
* to the goal we are looking for, our caller can delete it. In the
* inductive case where the node is an interior node, we'll likewise
* return whether the interior node is non-empty. If it is empty
* (because we just deleted its last child), then our caller can
* likewise delete it.
*/
template<typename G>
static bool removeGoal(std::shared_ptr<G> goal, std::weak_ptr<G> & gp)
{
for (auto i = goalMap.begin(); i != goalMap.end();)
if (!f(i->second))
return gp.lock() != goal;
}
template<typename K, typename G, typename Inner>
static bool removeGoal(std::shared_ptr<G> goal, std::map<K, Inner> & goalMap)
{
/* !!! inefficient */
for (auto i = goalMap.begin(); i != goalMap.end();) {
if (!removeGoal(goal, i->second))
i = goalMap.erase(i);
else ++i;
else
++i;
}
return !goalMap.empty();
}
template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
template<typename G>
static bool removeGoal(std::shared_ptr<G> goal, typename DerivedPathMap<std::map<OutputsSpec, std::weak_ptr<G>>>::ChildNode & node)
{
/* !!! inefficient */
cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
return gp.lock() != goal;
});
}
template<typename K>
static void removeGoal(std::shared_ptr<DerivationGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<DerivationGoal>>::ChildNode> & goalMap);
template<typename K>
static void removeGoal(std::shared_ptr<DerivationGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<DerivationGoal>>::ChildNode> & goalMap)
{
/* !!! inefficient */
cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<DerivationGoal>>::ChildNode & node) -> bool {
if (node.value.lock() == goal)
node.value.reset();
removeGoal(goal, node.childMap);
return !node.value.expired() || !node.childMap.empty();
});
return removeGoal(goal, node.value) || removeGoal(goal, node.childMap);
}
void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals.map);
if (auto drvGoal = std::dynamic_pointer_cast<DerivationTrampolineGoal>(goal))
nix::removeGoal(drvGoal, derivationTrampolineGoals.map);
else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else if (auto drvBuildingGoal = std::dynamic_pointer_cast<DerivationBuildingGoal>(goal))
nix::removeGoal(drvBuildingGoal, derivationBuildingGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
@ -312,13 +307,12 @@ void Worker::run(const Goals & _topGoals)
for (auto & i : _topGoals) {
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
if (auto goal = dynamic_cast<DerivationTrampolineGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built {
.drvPath = goal->drvReq,
.outputs = goal->wantedOutputs,
});
} else
if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Opaque{goal->storePath});
}
}