nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-03 15:40:59 +01:00
Code Activity

Merge remote-tracking branch 'upstream/master' into ca-drv-exotic

Browse source
This commit is contained in:
John Ericson 2023-04-17 18:10:12 -04:00
commit f56c4a5bdf
39 changed files with 708 additions and 206 deletions
Download patch file Download diff file Expand all files Collapse all files

93
src/libstore/build/derivation-goal.cc
View file

@ -145,8 +145,20 @@ void DerivationGoal::work()
void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
{
auto newWanted = wantedOutputs.union_(outputs);
if (!newWanted.isSubsetOf(wantedOutputs))
needRestart = true;
switch (needRestart) {
case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed:
if (!newWanted.isSubsetOf(wantedOutputs))
needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed;
break;
case NeedRestartForMoreOutputs::OutputsAddedDoNeed:
/* No need to check whether we added more outputs, because a
restart is already queued up. */
break;
case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed:
/* We are already building all outputs, so it doesn't matter if
we now want more. */
break;
};
wantedOutputs = newWanted;
}
@ -297,12 +309,29 @@ void DerivationGoal::outputsSubstitutionTried()
In particular, it may be the case that the hole in the closure is
an output of the current derivation, which causes a loop if retried.
*/
if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true;
{
bool substitutionFailed =
nrIncompleteClosure > 0 &&
nrIncompleteClosure == nrFailed;
switch (retrySubstitution) {
case RetrySubstitution::NoNeed:
if (substitutionFailed)
retrySubstitution = RetrySubstitution::YesNeed;
break;
case RetrySubstitution::YesNeed:
// Should not be able to reach this state from here.
assert(false);
break;
case RetrySubstitution::AlreadyRetried:
debug("substitution failed again, but we already retried once. Not retrying again.");
break;
}
}
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
if (needRestart) {
needRestart = false;
if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) {
needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
haveDerivation();
return;
}
@ -330,6 +359,10 @@ void DerivationGoal::outputsSubstitutionTried()
produced using a substitute. So we have to build instead. */
void DerivationGoal::gaveUpOnSubstitution()
{
/* At this point we are building all outputs, so if more are wanted there
is no need to restart. */
needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed;
/* The inputs must be built before we can build this goal. */
inputDrvOutputs.clear();
if (useDerivation)
@ -451,8 +484,8 @@ void DerivationGoal::inputsRealised()
return;
}
if (retrySubstitution && !retriedSubstitution) {
retriedSubstitution = true;
if (retrySubstitution == RetrySubstitution::YesNeed) {
retrySubstitution = RetrySubstitution::AlreadyRetried;
haveDerivation();
return;
}
@ -570,8 +603,6 @@ void DerivationGoal::inputsRealised()
build hook. */
state = &DerivationGoal::tryToBuild;
worker.wakeUp(shared_from_this());
buildResult = BuildResult { .path = buildResult.path };
}
void DerivationGoal::started()
@ -982,7 +1013,7 @@ void DerivationGoal::resolvedFinished()
auto resolvedDrv = *resolvedDrvGoal->drv;
auto & resolvedResult = resolvedDrvGoal->buildResult;
DrvOutputs builtOutputs;
SingleDrvOutputs builtOutputs;
if (resolvedResult.success()) {
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
@ -1008,7 +1039,7 @@ void DerivationGoal::resolvedFinished()
worker.store.printStorePath(drvPath), wantedOutput);
auto realisation = [&]{
auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput });
auto take1 = get(resolvedResult.builtOutputs, wantedOutput);
if (take1) return *take1;
/* The above `get` should work. But sateful tracking of
@ -1033,7 +1064,7 @@ void DerivationGoal::resolvedFinished()
worker.store.registerDrvOutput(newRealisation);
}
outputPaths.insert(realisation.outPath);
builtOutputs.emplace(realisation.id, realisation);
builtOutputs.emplace(wantedOutput, realisation);
}
runPostBuildHook(
@ -1158,7 +1189,7 @@ HookReply DerivationGoal::tryBuildHook()
}
DrvOutputs DerivationGoal::registerOutputs()
SingleDrvOutputs DerivationGoal::registerOutputs()
{
/* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have
@ -1320,7 +1351,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
}
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
{
if (!drv->type().isPure()) return { false, {} };
@ -1333,7 +1364,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
return static_cast<StringSet>(names);
},
}, wantedOutputs.raw());
DrvOutputs validOutputs;
SingleDrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap()) {
auto initialOutput = get(initialOutputs, i.first);
@ -1376,7 +1407,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
}
}
if (info.wanted && info.known && info.known->isValid())
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
}
// If we requested all the outputs, we are always fine.
@ -1400,7 +1431,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
}
DrvOutputs DerivationGoal::assertPathValidity()
SingleDrvOutputs DerivationGoal::assertPathValidity()
{
auto [allValid, validOutputs] = checkPathValidity();
if (!allValid)
@ -1411,7 +1442,7 @@ DrvOutputs DerivationGoal::assertPathValidity()
void DerivationGoal::done(
BuildResult::Status status,
DrvOutputs builtOutputs,
SingleDrvOutputs builtOutputs,
std::optional<Error> ex)
{
buildResult.status = status;
@ -1452,12 +1483,28 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
{
Goal::waiteeDone(waitee, result);
if (waitee->buildResult.success())
if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
if (!useDerivation) return;
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
if (!dg) return;
auto outputs = fullDrv.inputDrvs.find(dg->drvPath);
if (outputs == fullDrv.inputDrvs.end()) return;
for (auto & outputName : outputs->second) {
auto buildResult = dg->getBuildResult(DerivedPath::Built {
.drvPath = dg->drvPath,
.outputs = OutputsSpec::Names { outputName },
});
if (buildResult.success()) {
auto i = buildResult.builtOutputs.find(outputName);
if (i != buildResult.builtOutputs.end())
inputDrvOutputs.insert_or_assign(
{ bfd->drvPath, output.outputName },
realisation.outPath);
{ dg->drvPath, outputName },
i->second.outPath);
}
}
}
}

64
src/libstore/build/derivation-goal.hh
View file

@ -78,22 +78,58 @@ struct DerivationGoal : public Goal
*/
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/**
* See `needRestart`; just for that field.
*/
enum struct NeedRestartForMoreOutputs {
/**
* The goal state machine is progressing based on the current value of
* `wantedOutputs. No actions are needed.
*/
OutputsUnmodifedDontNeed,
/**
* `wantedOutputs` has been extended, but the state machine is
* proceeding according to its old value, so we need to restart.
*/
OutputsAddedDoNeed,
/**
* The goal state machine has progressed to the point of doing a build,
* in which case all outputs will be produced, so extensions to
* `wantedOutputs` no longer require a restart.
*/
BuildInProgressWillNotNeed,
};
/**
* Whether additional wanted outputs have been added.
*/
bool needRestart = false;
NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
/**
* See `retrySubstitution`; just for that field.
*/
enum RetrySubstitution {
/**
* No issues have yet arose, no need to restart.
*/
NoNeed,
/**
* Something failed and there is an incomplete closure. Let's retry
* substituting.
*/
YesNeed,
/**
* We are current or have already retried substitution, and whether or
* not something goes wrong we will not retry again.
*/
AlreadyRetried,
};
/**
* Whether to retry substituting the outputs after building the
* inputs. This is done in case of an incomplete closure.
*/
bool retrySubstitution = false;
/**
* Whether we've retried substitution, in which case we won't try
* again.
*/
bool retriedSubstitution = false;
RetrySubstitution retrySubstitution = RetrySubstitution::NoNeed;
/**
* The derivation stored at drvPath.
@ -217,7 +253,7 @@ struct DerivationGoal : public Goal
* Check that the derivation outputs all exist and register them
* as valid.
*/
virtual DrvOutputs registerOutputs();
virtual SingleDrvOutputs registerOutputs();
/**
* Open a log file and a pipe to it.
@ -270,17 +306,17 @@ struct DerivationGoal : public Goal
* Update 'initialOutputs' to determine the current status of the
* outputs of the derivation. Also returns a Boolean denoting
* whether all outputs are valid and non-corrupt, and a
* 'DrvOutputs' structure containing the valid and wanted
* 'SingleDrvOutputs' structure containing the valid and wanted
* outputs.
*/
std::pair<bool, DrvOutputs> checkPathValidity();
std::pair<bool, SingleDrvOutputs> checkPathValidity();
/**
* Aborts if any output is not valid or corrupt, and otherwise
* returns a 'DrvOutputs' structure containing the wanted
* returns a 'SingleDrvOutputs' structure containing the wanted
* outputs.
*/
DrvOutputs assertPathValidity();
SingleDrvOutputs assertPathValidity();
/**
* Forcibly kill the child process, if any.
@ -293,7 +329,7 @@ struct DerivationGoal : public Goal
void done(
BuildResult::Status status,
DrvOutputs builtOutputs = {},
SingleDrvOutputs builtOutputs = {},
std::optional<Error> ex = {});
void waiteeDone(GoalPtr waitee, ExitCode result) override;

47
src/libstore/build/entry-points.cc
View file

@ -10,16 +10,8 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
Worker worker(*this, evalStore ? *evalStore : *this);
Goals goals;
for (const auto & br : reqs) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
},
[&](const DerivedPath::Opaque & bo) {
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
}
for (auto & br : reqs)
goals.insert(worker.makeGoal(br, buildMode));
worker.run(goals);
@ -47,7 +39,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
}
}
std::vector<BuildResult> Store::buildPathsWithResults(
std::vector<KeyedBuildResult> Store::buildPathsWithResults(
const std::vector<DerivedPath> & reqs,
BuildMode buildMode,
std::shared_ptr<Store> evalStore)
@ -55,23 +47,23 @@ std::vector<BuildResult> Store::buildPathsWithResults(
Worker worker(*this, evalStore ? *evalStore : *this);
Goals goals;
for (const auto & br : reqs) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
},
[&](const DerivedPath::Opaque & bo) {
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
std::vector<std::pair<const DerivedPath &, GoalPtr>> state;
for (const auto & req : reqs) {
auto goal = worker.makeGoal(req, buildMode);
goals.insert(goal);
state.push_back({req, goal});
}
worker.run(goals);
std::vector<BuildResult> results;
std::vector<KeyedBuildResult> results;
for (auto & i : goals)
results.push_back(i->buildResult);
for (auto & [req, goalPtr] : state)
results.emplace_back(KeyedBuildResult {
goalPtr->getBuildResult(req),
/* .path = */ req,
});
return results;
}
@ -84,15 +76,14 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
try {
worker.run(Goals{goal});
return goal->buildResult;
return goal->getBuildResult(DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All {},
});
} catch (Error & e) {
return BuildResult {
.status = BuildResult::MiscFailure,
.errorMsg = e.msg(),
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
};
};
}

23
src/libstore/build/goal.cc
View file

@ -11,6 +11,29 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
}
BuildResult Goal::getBuildResult(const DerivedPath & req) {
BuildResult res { buildResult };
if (auto pbp = std::get_if<DerivedPath::Built>(&req)) {
auto & bp = *pbp;
/* Because goals are in general shared between derived paths
that share the same derivation, we need to filter their
results to get back just the results we care about.
*/
for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) {
if (bp.outputs.contains(it->first))
++it;
else
it = res.builtOutputs.erase(it);
}
}
return res;
}
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
{
if (goals.find(p) != goals.end())

16
src/libstore/build/goal.hh
View file

@ -81,11 +81,26 @@ struct Goal : public std::enable_shared_from_this<Goal>
*/
ExitCode exitCode = ecBusy;
protected:
/**
* Build result.
*/
BuildResult buildResult;
public:
/**
* Project a `BuildResult` with just the information that pertains
* to the given request.
*
* In general, goals may be aliased between multiple requests, and
* the stored `BuildResult` has information for the union of all
* requests. We don't want to leak what the other request are for
* sake of both privacy and determinism, and this "safe accessor"
* ensures we don't.
*/
BuildResult getBuildResult(const DerivedPath &);
/**
* Exception containing an error message, if any.
*/
@ -93,7 +108,6 @@ struct Goal : public std::enable_shared_from_this<Goal>
Goal(Worker & worker, DerivedPath path)
: worker(worker)
, buildResult { .path = std::move(path) }
{ }
virtual ~Goal()

8
src/libstore/build/local-derivation-goal.cc
View file

@ -1335,7 +1335,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
result.rethrow();
}
std::vector<BuildResult> buildPathsWithResults(
std::vector<KeyedBuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr) override
@ -2174,7 +2174,7 @@ void LocalDerivationGoal::runChild()
}
DrvOutputs LocalDerivationGoal::registerOutputs()
SingleDrvOutputs LocalDerivationGoal::registerOutputs()
{
/* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have
@ -2695,7 +2695,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
means it's safe to link the derivation to the output hash. We must do
that for floating CA derivations, which otherwise couldn't be cached,
but it's fine to do in all cases. */
DrvOutputs builtOutputs;
SingleDrvOutputs builtOutputs;
for (auto & [outputName, newInfo] : infos) {
auto oldinfo = get(initialOutputs, outputName);
@ -2714,7 +2714,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
worker.store.registerDrvOutput(thisRealisation);
}
if (wantedOutputs.contains(outputName))
builtOutputs.emplace(thisRealisation.id, thisRealisation);
builtOutputs.emplace(outputName, thisRealisation);
}
return builtOutputs;

2
src/libstore/build/local-derivation-goal.hh
View file

@ -237,7 +237,7 @@ struct LocalDerivationGoal : public DerivationGoal
* Check that the derivation outputs all exist and register them
* as valid.
*/
DrvOutputs registerOutputs() override;
SingleDrvOutputs registerOutputs() override;
void signRealisation(Realisation &) override;

15
src/libstore/build/worker.cc
View file

@ -92,6 +92,7 @@ std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const Sto
return goal;
}
std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
{
std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
@ -104,6 +105,20 @@ std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal
return goal;
}
GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr {
return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode);
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
},
}, req.raw());
}
template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{

12
src/libstore/build/worker.hh
View file

@ -181,7 +181,7 @@ public:
*/
/**
* derivation goal
* @ref DerivationGoal "derivation goal"
*/
private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
@ -196,11 +196,19 @@ public:
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
/**
* substitution goal
* @ref SubstitutionGoal "substitution goal"
*/
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/**
* Make a goal corresponding to the `DerivedPath`.
*
* It will be a `DerivationGoal` for a `DerivedPath::Built` or
* a `SubstitutionGoal` for a `DerivedPath::Opaque`.
*/
GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal);
/**
* Remove a dead goal.
*/