nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-05 08:31:00 +01:00
Code Activity

Revert "Revert "Adapt scheduler to work with dynamic derivations""

Browse source 
This fixes dynamic derivations, reverting #9081.

I believe that this time around, #9052 is fixed. When I first rebased
this, tests were failing (which wasn't the case before). The cause of
those test failures were due to the crude job in which the outer goal
tried to exit with the inner goal's status.

Now, that error handling has been reworked to be more faithful. The exit
exit status and exception of the inner goal is returned by the outer
goal. The exception was what was causing the test failures, but I
believe it was not having the right error code (there is more than one
for failure) that caused #9081.

The only cost of doing things the "right way" was that I had to
introduce a hacky `preserveException` boolean. I don't like this, but,
then again, none of us like anything about how the scheduler works.
Issue #11927 is still there to clean everything up, subsuming the need
for any `preserveException` because I doubt we will be fishing
information out of state machines like this at all.

This reverts commit 8440afbed7.

Co-Authored-By: Eelco Dolstra <edolstra@gmail.com>
This commit is contained in:
John Ericson 2023-11-20 16:38:41 -05:00
parent a562d0b6ce
commit c98525235f
17 changed files with 397 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

92
src/libstore/build/worker.cc
View file

@ -5,6 +5,7 @@
#include "drv-output-substitution-goal.hh"
#include "derivation-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows
# include "derivation-creation-and-realisation-goal.hh"
# include "local-derivation-goal.hh"
# include "hook-instance.hh"
#endif
@ -43,6 +44,24 @@ Worker::~Worker()
}
std::shared_ptr<DerivationCreationAndRealisationGoal> Worker::makeDerivationCreationAndRealisationGoal(
ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs,
BuildMode buildMode)
{
std::weak_ptr<DerivationCreationAndRealisationGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
std::shared_ptr<DerivationCreationAndRealisationGoal> goal = goal_weak.lock();
if (!goal) {
goal = std::make_shared<DerivationCreationAndRealisationGoal>(drvReq, wantedOutputs, *this, buildMode);
goal_weak = goal;
wakeUp(goal);
} else {
goal->addWantedOutputs(wantedOutputs);
}
return goal;
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath,
const OutputsSpec & wantedOutputs,
@ -120,10 +139,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr {
if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
else
throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
return makeDerivationCreationAndRealisationGoal(bfd.drvPath, bfd.outputs, buildMode);
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@ -132,24 +148,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
}
template<typename K, typename V, typename F>
static void cullMap(std::map<K, V> & goalMap, F f)
{
for (auto i = goalMap.begin(); i != goalMap.end();)
if (!f(i->second))
i = goalMap.erase(i);
else ++i;
}
template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{
/* !!! inefficient */
for (auto i = goalMap.begin();
i != goalMap.end(); )
if (i->second.lock() == goal) {
auto j = i; ++j;
goalMap.erase(i);
i = j;
}
else ++i;
cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
return gp.lock() != goal;
});
}
template<typename K>
static void removeGoal(std::shared_ptr<DerivationCreationAndRealisationGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<DerivationCreationAndRealisationGoal>>::ChildNode> & goalMap);
template<typename K>
static void removeGoal(std::shared_ptr<DerivationCreationAndRealisationGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<DerivationCreationAndRealisationGoal>>::ChildNode> & goalMap)
{
/* !!! inefficient */
cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<DerivationCreationAndRealisationGoal>>::ChildNode & node) -> bool {
if (node.value.lock() == goal)
node.value.reset();
removeGoal(goal, node.childMap);
return !node.value.expired() || !node.childMap.empty();
});
}
void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
if (auto drvGoal = std::dynamic_pointer_cast<DerivationCreationAndRealisationGoal>(goal))
nix::removeGoal(drvGoal, outerDerivationGoals.map);
else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else
if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
@ -215,6 +253,9 @@ void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::Com
case JobCategory::Build:
nrLocalBuilds++;
break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default:
unreachable();
}
@ -238,6 +279,9 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
assert(nrLocalBuilds > 0);
nrLocalBuilds--;
break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default:
unreachable();
}
@ -290,9 +334,9 @@ void Worker::run(const Goals & _topGoals)
for (auto & i : _topGoals) {
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
if (auto goal = dynamic_cast<DerivationCreationAndRealisationGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(goal->drvPath),
.drvPath = goal->drvReq,
.outputs = goal->wantedOutputs,
});
} else
@ -552,4 +596,22 @@ GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
return subGoal;
}
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
{
return subGoal;
}
std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
{
auto * odg = dynamic_cast<DerivationCreationAndRealisationGoal *>(&*waitee);
if (!odg) return std::nullopt;
/* If we failed to obtain the concrete drv, we won't have created
the concrete derivation goal. */
if (!odg->concreteDrvGoal) return std::nullopt;
return {{
std::cref(*odg->concreteDrvGoal),
std::cref(*odg->drvReq),
}};
}
}