nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-02 23:20:59 +01:00
Code Activity

Merge remote-tracking branch 'detsys/detsys-main' into logger-improvements

Browse source
This commit is contained in:
Eelco Dolstra 2025-03-26 23:21:32 +01:00
commit c8692b378b
206 changed files with 4776 additions and 2018 deletions
Download patch file Download diff file Expand all files Collapse all files

309
src/libstore/build/derivation-goal.cc
View file

@ -36,14 +36,6 @@
namespace nix {
Goal::Co DerivationGoal::init() {
if (useDerivation) {
co_return getDerivation();
} else {
co_return haveDerivation();
}
}
DerivationGoal::DerivationGoal(const StorePath & drvPath,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs })
@ -141,50 +133,44 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
}
Goal::Co DerivationGoal::getDerivation()
{
Goal::Co DerivationGoal::init() {
trace("init");
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
co_return loadDerivation();
}
if (useDerivation) {
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
co_await Suspend{};
co_return loadDerivation();
}
Goal::Co DerivationGoal::loadDerivation()
{
trace("loading derivation");
if (nrFailed != 0) {
co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
}
/* `drvPath' should already be a root, but let's be on the safe
side: if the user forgot to make it a root, we wouldn't want
things being garbage collected while we're busy. */
worker.evalStore.addTempRoot(drvPath);
/* Get the derivation. It is probably in the eval store, but it might be inthe main store:
- Resolved derivation are resolved against main store realisations, and so must be stored there.
- Dynamic derivations are built, and so are found in the main store.
*/
for (auto * drvStore : { &worker.evalStore, &worker.store }) {
if (drvStore->isValidPath(drvPath)) {
drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
break;
if (buildMode != bmNormal || !worker.evalStore.isValidPath(drvPath)) {
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
co_await Suspend{};
}
trace("loading derivation");
if (nrFailed != 0) {
co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
}
/* `drvPath' should already be a root, but let's be on the safe
side: if the user forgot to make it a root, we wouldn't want
things being garbage collected while we're busy. */
worker.evalStore.addTempRoot(drvPath);
/* Get the derivation. It is probably in the eval store, but it might be inthe main store:
- Resolved derivation are resolved against main store realisations, and so must be stored there.
- Dynamic derivations are built, and so are found in the main store.
*/
for (auto * drvStore : { &worker.evalStore, &worker.store }) {
if (drvStore->isValidPath(drvPath)) {
drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
break;
}
}
assert(drv);
}
assert(drv);
co_return haveDerivation();
}
@ -195,58 +181,64 @@ Goal::Co DerivationGoal::haveDerivation()
trace("have derivation");
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
drvOptions = std::make_unique<DerivationOptions>(DerivationOptions::fromParsedDerivation(*parsedDrv));
if (!drv->type().hasKnownOutputPaths())
experimentalFeatureSettings.require(Xp::CaDerivations);
if (drv->type().isImpure()) {
experimentalFeatureSettings.require(Xp::ImpureDerivations);
for (auto & [outputName, output] : drv->outputs) {
auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
assert(!worker.store.isValidPath(randomPath));
initialOutputs.insert({
outputName,
InitialOutput {
.wanted = true,
.outputHash = impureOutputHash,
.known = InitialOutputStatus {
.path = randomPath,
.status = PathStatus::Absent
}
}
});
}
co_return gaveUpOnSubstitution();
}
for (auto & i : drv->outputsAndOptPaths(worker.store))
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
for (auto & [outputName, outputHash] : outputHashes)
initialOutputs.insert({
outputName,
InitialOutput {
{
bool impure = drv->type().isImpure();
if (impure) experimentalFeatureSettings.require(Xp::ImpureDerivations);
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
for (auto & [outputName, outputHash] : outputHashes) {
InitialOutput v{
.wanted = true, // Will be refined later
.outputHash = outputHash
};
/* TODO we might want to also allow randomizing the paths
for regular CA derivations, e.g. for sake of checking
determinism. */
if (impure) {
v.known = InitialOutputStatus {
.path = StorePath::random(outputPathName(drv->name, outputName)),
.status = PathStatus::Absent,
};
}
});
/* Check what outputs paths are not already valid. */
auto [allValid, validOutputs] = checkPathValidity();
initialOutputs.insert({
outputName,
std::move(v),
});
}
/* If they are all valid, then we're done. */
if (allValid && buildMode == bmNormal) {
co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
if (impure) {
/* We don't yet have any safe way to cache an impure derivation at
this step. */
co_return gaveUpOnSubstitution();
}
}
{
/* Check what outputs paths are not already valid. */
auto [allValid, validOutputs] = checkPathValidity();
/* If they are all valid, then we're done. */
if (allValid && buildMode == bmNormal) {
co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
}
}
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
if (settings.useSubstitutes && drvOptions->substitutesAllowed())
for (auto & [outputName, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known)
@ -268,12 +260,7 @@ Goal::Co DerivationGoal::haveDerivation()
}
if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */
co_return outputsSubstitutionTried();
}
Goal::Co DerivationGoal::outputsSubstitutionTried()
{
trace("all outputs substituted (maybe)");
assert(!drv->type().isImpure());
@ -399,84 +386,7 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
}
if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */
co_return inputsRealised();
}
Goal::Co DerivationGoal::repairClosure()
{
assert(!drv->type().isImpure());
/* If we're repairing, we now know that our own outputs are valid.
Now check whether the other paths in the outputs closure are
good. If not, then start derivation goals for the derivations
that produced those outputs. */
/* Get the output closure. */
auto outputs = queryDerivationOutputMap();
StorePathSet outputClosure;
for (auto & i : outputs) {
if (!wantedOutputs.contains(i.first)) continue;
worker.store.computeFSClosure(i.second, outputClosure);
}
/* Filter out our own outputs (which we have already checked). */
for (auto & i : outputs)
outputClosure.erase(i.second);
/* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output
closure. */
StorePathSet inputClosure;
if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
std::map<StorePath, StorePath> outputsToDrv;
for (auto & i : inputClosure)
if (i.isDerivation()) {
auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
for (auto & j : depOutputs)
if (j.second)
outputsToDrv.insert_or_assign(*j.second, i);
}
/* Check each path (slow!). */
for (auto & i : outputClosure) {
if (worker.pathContentsGood(i)) continue;
printError(
"found corrupted or missing path '%s' in the output closure of '%s'",
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
auto drvPath2 = outputsToDrv.find(i);
if (drvPath2 == outputsToDrv.end())
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else
addWaitee(worker.makeGoal(
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath2->second),
.outputs = OutputsSpec::All { },
},
bmRepair));
}
if (waitees.empty()) {
co_return done(BuildResult::AlreadyValid, assertPathValidity());
} else {
co_await Suspend{};
co_return closureRepaired();
}
}
Goal::Co DerivationGoal::closureRepaired()
{
trace("closure repaired");
if (nrFailed > 0)
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
worker.store.printStorePath(drvPath));
co_return done(BuildResult::AlreadyValid, assertPathValidity());
}
Goal::Co DerivationGoal::inputsRealised()
{
trace("all inputs realised");
if (nrFailed != 0) {
@ -718,7 +628,7 @@ Goal::Co DerivationGoal::tryToBuild()
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
bool buildLocally =
(buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store))
(buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv))
&& settings.maxBuildJobs.get() != 0;
if (!buildLocally) {
@ -766,6 +676,73 @@ Goal::Co DerivationGoal::tryLocalBuild() {
}
Goal::Co DerivationGoal::repairClosure()
{
assert(!drv->type().isImpure());
/* If we're repairing, we now know that our own outputs are valid.
Now check whether the other paths in the outputs closure are
good. If not, then start derivation goals for the derivations
that produced those outputs. */
/* Get the output closure. */
auto outputs = queryDerivationOutputMap();
StorePathSet outputClosure;
for (auto & i : outputs) {
if (!wantedOutputs.contains(i.first)) continue;
worker.store.computeFSClosure(i.second, outputClosure);
}
/* Filter out our own outputs (which we have already checked). */
for (auto & i : outputs)
outputClosure.erase(i.second);
/* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output
closure. */
StorePathSet inputClosure;
if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
std::map<StorePath, StorePath> outputsToDrv;
for (auto & i : inputClosure)
if (i.isDerivation()) {
auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
for (auto & j : depOutputs)
if (j.second)
outputsToDrv.insert_or_assign(*j.second, i);
}
/* Check each path (slow!). */
for (auto & i : outputClosure) {
if (worker.pathContentsGood(i)) continue;
printError(
"found corrupted or missing path '%s' in the output closure of '%s'",
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
auto drvPath2 = outputsToDrv.find(i);
if (drvPath2 == outputsToDrv.end())
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else
addWaitee(worker.makeGoal(
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath2->second),
.outputs = OutputsSpec::All { },
},
bmRepair));
}
if (waitees.empty()) {
co_return done(BuildResult::AlreadyValid, assertPathValidity());
} else {
co_await Suspend{};
trace("closure repaired");
if (nrFailed > 0)
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
worker.store.printStorePath(drvPath));
co_return done(BuildResult::AlreadyValid, assertPathValidity());
}
}
static void chmod_(const Path & path, mode_t mode)
{
if (chmod(path.c_str(), mode) == -1)
@ -1145,7 +1122,7 @@ HookReply DerivationGoal::tryBuildHook()
<< (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
<< drv->platform
<< worker.store.printStorePath(drvPath)
<< parsedDrv->getRequiredSystemFeatures();
<< drvOptions->getRequiredSystemFeatures(*drv);
worker.hook->sink.flush();
/* Read the first line of input, which should be a word indicating
@ -1247,7 +1224,7 @@ SingleDrvOutputs DerivationGoal::registerOutputs()
to do anything here.
We can only early return when the outputs are known a priori. For
floating content-addressed derivations this isn't the case.
floating content-addressing derivations this isn't the case.
*/
return assertPathValidity();
}

9
src/libstore/build/derivation-goal.hh
View file

@ -2,6 +2,7 @@
///@file
#include "parsed-derivations.hh"
#include "derivation-options.hh"
#ifndef _WIN32
# include "user-lock.hh"
#endif
@ -80,7 +81,7 @@ struct DerivationGoal : public Goal
/**
* Mapping from input derivations + output names to actual store
* paths. This is filled in by waiteeDone() as each dependency
* finishes, before inputsRealised() is reached.
* finishes, before `trace("all inputs realised")` is reached.
*/
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
@ -143,6 +144,7 @@ struct DerivationGoal : public Goal
std::unique_ptr<Derivation> drv;
std::unique_ptr<ParsedDerivation> parsedDrv;
std::unique_ptr<DerivationOptions> drvOptions;
/**
* The remainder is state held during the build.
@ -233,13 +235,8 @@ struct DerivationGoal : public Goal
* The states.
*/
Co init() override;
Co getDerivation();
Co loadDerivation();
Co haveDerivation();
Co outputsSubstitutionTried();
Co gaveUpOnSubstitution();
Co closureRepaired();
Co inputsRealised();
Co tryToBuild();
virtual Co tryLocalBuild();
Co buildDone();