nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-03 07:31:00 +01:00
Code Activity

Merge remote-tracking branch 'upstream/master' into best-effort-supplementary-groups

Browse source
This commit is contained in:
John Ericson 2023-05-15 16:50:11 -04:00
commit 746c6aae3f
75 changed files with 1544 additions and 519 deletions
Download patch file Download diff file Expand all files Collapse all files

41
src/libstore/build/derivation-goal.cc
View file

@ -274,11 +274,13 @@ void DerivationGoal::haveDerivation()
)
)
);
else
else {
auto * cap = getDerivationCA(*drv);
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
status.known->path,
buildMode == bmRepair ? Repair : NoRepair,
getDerivationCA(*drv))));
cap ? std::optional { *cap } : std::nullopt)));
}
}
if (waitees.empty()) /* to prevent hang (no wake-up event) */
@ -1020,43 +1022,33 @@ void DerivationGoal::resolvedFinished()
StorePathSet outputPaths;
// `wantedOutputs` might merely indicate “all the outputs”
auto realWantedOutputs = std::visit(overloaded {
[&](const OutputsSpec::All &) {
return resolvedDrv.outputNames();
},
[&](const OutputsSpec::Names & names) {
return static_cast<std::set<std::string>>(names);
},
}, wantedOutputs.raw());
for (auto & wantedOutput : realWantedOutputs) {
auto initialOutput = get(initialOutputs, wantedOutput);
auto resolvedHash = get(resolvedHashes, wantedOutput);
for (auto & outputName : resolvedDrv.outputNames()) {
auto initialOutput = get(initialOutputs, outputName);
auto resolvedHash = get(resolvedHashes, outputName);
if ((!initialOutput) || (!resolvedHash))
throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
worker.store.printStorePath(drvPath), wantedOutput);
worker.store.printStorePath(drvPath), outputName);
auto realisation = [&]{
auto take1 = get(resolvedResult.builtOutputs, wantedOutput);
auto take1 = get(resolvedResult.builtOutputs, outputName);
if (take1) return *take1;
/* The above `get` should work. But sateful tracking of
outputs in resolvedResult, this can get out of sync with the
store, which is our actual source of truth. For now we just
check the store directly if it fails. */
auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput });
auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName });
if (take2) return *take2;
throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName);
}();
if (drv->type().isPure()) {
auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
newRealisation.signatures.clear();
if (!drv->type().isFixed())
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
@ -1064,7 +1056,7 @@ void DerivationGoal::resolvedFinished()
worker.store.registerDrvOutput(newRealisation);
}
outputPaths.insert(realisation.outPath);
builtOutputs.emplace(wantedOutput, realisation);
builtOutputs.emplace(outputName, realisation);
}
runPostBuildHook(
@ -1406,7 +1398,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
);
}
}
if (info.wanted && info.known && info.known->isValid())
if (info.known && info.known->isValid())
validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
}
@ -1457,8 +1449,9 @@ void DerivationGoal::done(
mcRunningBuilds.reset();
if (buildResult.success()) {
assert(!builtOutputs.empty());
buildResult.builtOutputs = std::move(builtOutputs);
auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs));
assert(!wantedBuiltOutputs.empty());
buildResult.builtOutputs = std::move(wantedBuiltOutputs);
if (status == BuildResult::Built)
worker.doneBuilds++;
} else {

8
src/libstore/build/derivation-goal.hh
View file

@ -306,15 +306,13 @@ struct DerivationGoal : public Goal
* Update 'initialOutputs' to determine the current status of the
* outputs of the derivation. Also returns a Boolean denoting
* whether all outputs are valid and non-corrupt, and a
* 'SingleDrvOutputs' structure containing the valid and wanted
* outputs.
* 'SingleDrvOutputs' structure containing the valid outputs.
*/
std::pair<bool, SingleDrvOutputs> checkPathValidity();
/**
* Aborts if any output is not valid or corrupt, and otherwise
* returns a 'SingleDrvOutputs' structure containing the wanted
* outputs.
* returns a 'SingleDrvOutputs' structure containing all outputs.
*/
SingleDrvOutputs assertPathValidity();
@ -335,6 +333,8 @@ struct DerivationGoal : public Goal
void waiteeDone(GoalPtr waitee, ExitCode result) override;
StorePathSet exportReferences(const StorePathSet & storePaths);
JobCategory jobCategory() override { return JobCategory::Build; };
};
MakeError(NotDeterministic, BuildError);

4
src/libstore/build/drv-output-substitution-goal.hh
View file

@ -21,7 +21,7 @@ class Worker;
class DrvOutputSubstitutionGoal : public Goal {
/**
* The drv output we're trying to substitue
* The drv output we're trying to substitute
*/
DrvOutput id;
@ -72,6 +72,8 @@ public:
void work() override;
void handleEOF(int fd) override;
JobCategory jobCategory() override { return JobCategory::Substitution; };
};
}

13
src/libstore/build/goal.hh
View file

@ -34,6 +34,17 @@ typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
*/
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
/**
* Used as a hint to the worker on how to schedule a particular goal. For example,
* builds are typically CPU- and memory-bound, while substitutions are I/O bound.
* Using this information, the worker might decide to schedule more or fewer goals
* of each category in parallel.
*/
enum struct JobCategory {
Build,
Substitution,
};
struct Goal : public std::enable_shared_from_this<Goal>
{
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
@ -150,6 +161,8 @@ public:
void amDone(ExitCode result, std::optional<Error> ex = {});
virtual void cleanup() { }
virtual JobCategory jobCategory() = 0;
};
void addToWeakGoals(WeakGoals & goals, GoalPtr p);

63
src/libstore/build/local-derivation-goal.cc
View file

@ -2421,37 +2421,51 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
throw BuildError(
"output path %1% without valid stats info",
actualPath);
if (outputHash.method == FileIngestionMethod::Flat) {
if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
{
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
throw BuildError(
"output path '%1%' should be a non-executable regular file "
"since recursive hashing is not enabled (outputHashMode=flat)",
"since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
actualPath);
}
rewriteOutput();
/* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath->hashPart() };
HashModuloSink caSink { outputHash.hashType, oldHashPart };
switch (outputHash.method) {
case FileIngestionMethod::Recursive:
dumpPath(actualPath, caSink);
break;
case FileIngestionMethod::Flat:
readFile(actualPath, caSink);
break;
}
std::visit(overloaded {
[&](const TextIngestionMethod &) {
readFile(actualPath, caSink);
},
[&](const FileIngestionMethod & m2) {
switch (m2) {
case FileIngestionMethod::Recursive:
dumpPath(actualPath, caSink);
break;
case FileIngestionMethod::Flat:
readFile(actualPath, caSink);
break;
}
},
}, outputHash.method.raw);
auto got = caSink.finish().first;
auto optCA = ContentAddressWithReferences::fromPartsOpt(
outputHash.method,
std::move(got),
rewriteRefs());
if (!optCA) {
// TODO track distinct failure modes separately (at the time of
// writing there is just one but `nullopt` is unclear) so this
// message can't get out of sync.
throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
}
ValidPathInfo newInfo0 {
worker.store,
outputPathName(drv->name, outputName),
FixedOutputInfo {
.hash = {
.method = outputHash.method,
.hash = got,
},
.references = rewriteRefs(),
},
*std::move(optCA),
Hash::dummy,
};
if (*scratchPath != newInfo0.path) {
@ -2498,13 +2512,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
},
[&](const DerivationOutput::CAFixed & dof) {
auto wanted = dof.ca.getHash();
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
.method = dof.hash.method,
.hashType = dof.hash.hash.type,
.method = dof.ca.getMethod(),
.hashType = wanted.type,
});
/* Check wanted hash */
const Hash & wanted = dof.hash.hash;
assert(newInfo0.ca);
auto got = newInfo0.ca->getHash();
if (wanted != got) {
@ -2517,6 +2532,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
wanted.to_string(SRI, true),
got.to_string(SRI, true)));
}
if (!newInfo0.references.empty())
delayedException = std::make_exception_ptr(
BuildError("illegal path references in fixed-output derivation '%s'",
worker.store.printStorePath(drvPath)));
return newInfo0;
},
@ -2696,8 +2716,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}
if (wantedOutputs.contains(outputName))
builtOutputs.emplace(outputName, thisRealisation);
builtOutputs.emplace(outputName, thisRealisation);
}
return builtOutputs;

9
src/libstore/build/substitution-goal.cc
View file

@ -200,11 +200,10 @@ void PathSubstitutionGoal::tryToRun()
{
trace("trying to run");
/* Make sure that we are allowed to start a build. Note that even
if maxBuildJobs == 0 (no local builds allowed), we still allow
a substituter to run. This is because substitutions cannot be
distributed to another machine via the build hook. */
if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
/* Make sure that we are allowed to start a substitution. Note that even
if maxSubstitutionJobs == 0, we still allow a substituter to run. This
prevents infinite waiting. */
if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
worker.waitForBuildSlot(shared_from_this());
return;
}

2
src/libstore/build/substitution-goal.hh
View file

@ -115,6 +115,8 @@ public:
void handleEOF(int fd) override;
void cleanup() override;
JobCategory jobCategory() override { return JobCategory::Substitution; };
};
}

25
src/libstore/build/worker.cc
View file

@ -18,6 +18,7 @@ Worker::Worker(Store & store, Store & evalStore)
{
/* Debugging: prevent recursive workers. */
nrLocalBuilds = 0;
nrSubstitutions = 0;
lastWokenUp = steady_time_point::min();
permanentFailure = false;
timedOut = false;
@ -176,6 +177,12 @@ unsigned Worker::getNrLocalBuilds()
}
unsigned Worker::getNrSubstitutions()
{
return nrSubstitutions;
}
void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
bool inBuildSlot, bool respectTimeouts)
{
@ -187,7 +194,10 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
child.inBuildSlot = inBuildSlot;
child.respectTimeouts = respectTimeouts;
children.emplace_back(child);
if (inBuildSlot) nrLocalBuilds++;
if (inBuildSlot) {
if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
else nrLocalBuilds++;
}
}
@ -198,8 +208,13 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
if (i == children.end()) return;
if (i->inBuildSlot) {
assert(nrLocalBuilds > 0);
nrLocalBuilds--;
if (goal->jobCategory() == JobCategory::Substitution) {
assert(nrSubstitutions > 0);
nrSubstitutions--;
} else {
assert(nrLocalBuilds > 0);
nrLocalBuilds--;
}
}
children.erase(i);
@ -220,7 +235,9 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
void Worker::waitForBuildSlot(GoalPtr goal)
{
debug("wait for build slot");
if (getNrLocalBuilds() < settings.maxBuildJobs)
bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution;
if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) ||
(isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))
wakeUp(goal); /* we can do it right away */
else
addToWeakGoals(wantingToBuild, goal);

19
src/libstore/build/worker.hh
View file

@ -88,11 +88,16 @@ private:
std::list<Child> children;
/**
* Number of build slots occupied. This includes local builds and
* substitutions but not remote builds via the build hook.
* Number of build slots occupied. This includes local builds but does not
* include substitutions or remote builds via the build hook.
*/
unsigned int nrLocalBuilds;
/**
* Number of substitution slots occupied.
*/
unsigned int nrSubstitutions;
/**
* Maps used to prevent multiple instantiations of a goal for the
* same derivation / path.
@ -220,12 +225,16 @@ public:
void wakeUp(GoalPtr goal);
/**
* Return the number of local build and substitution processes
* currently running (but not remote builds via the build
* hook).
* Return the number of local build processes currently running (but not
* remote builds via the build hook).
*/
unsigned int getNrLocalBuilds();
/**
* Return the number of substitution processes currently running.
*/
unsigned int getNrSubstitutions();
/**
* Registers a running child process. `inBuildSlot` means that
* the process counts towards the jobs limit.