nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-11 11:31:03 +01:00
Code Activity

Apply clang-format universally.

Browse source 
* It is tough to contribute to a project that doesn't use a formatter,
* It is extra hard to contribute to a project which has configured the formatter, but ignores it for some files
* Code formatting makes it harder to hide obscure / weird bugs by accident or on purpose,

Let's rip the bandaid off?

Note that PRs currently in flight should be able to be merged relatively easily by applying `clang-format` to their tip prior to merge.
This commit is contained in:
Graham Christensen 2025-07-18 12:47:27 -04:00
parent 41bf87ec70
commit e4f62e4608
587 changed files with 23258 additions and 23135 deletions
Download patch file Download diff file Expand all files Collapse all files

539
src/libstore/build/derivation-building-goal.cc
View file

File diff suppressed because it is too large Load diff

119
src/libstore/build/derivation-goal.cc
View file

@ -11,7 +11,7 @@
#include "nix/util/compression.hh"
#include "nix/store/common-protocol.hh"
#include "nix/store/common-protocol-impl.hh" // Don't remove is actually needed
#include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts
#include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts
#include <fstream>
#include <sys/types.h>
@ -24,8 +24,12 @@
namespace nix {
DerivationGoal::DerivationGoal(const StorePath & drvPath, const Derivation & drv,
const OutputName & wantedOutput, Worker & worker, BuildMode buildMode)
DerivationGoal::DerivationGoal(
const StorePath & drvPath,
const Derivation & drv,
const OutputName & wantedOutput,
Worker & worker,
BuildMode buildMode)
: Goal(worker, haveDerivation())
, drvPath(drvPath)
, wantedOutput(wantedOutput)
@ -33,17 +37,15 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const Derivation & drv
{
this->drv = std::make_unique<Derivation>(drv);
name = fmt(
"building of '%s' from in-memory derivation",
DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
name =
fmt("building of '%s' from in-memory derivation",
DerivedPath::Built{makeConstantStorePathRef(drvPath), drv.outputNames()}.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
worker.updateProgress();
}
std::string DerivationGoal::key()
{
/* Ensure that derivations get built in order of their name,
@ -56,7 +58,6 @@ std::string DerivationGoal::key()
}.to_string(worker.store);
}
Goal::Co DerivationGoal::haveDerivation()
{
trace("have derivation");
@ -76,8 +77,7 @@ Goal::Co DerivationGoal::haveDerivation()
/* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */
auto gaveUpOnSubstitution = [&]() -> Goal::Co
{
auto gaveUpOnSubstitution = [&]() -> Goal::Co {
auto g = worker.makeDerivationBuildingGoal(drvPath, *drv, buildMode);
/* We will finish with it ourselves, as if we were the derivational goal. */
@ -103,7 +103,7 @@ Goal::Co DerivationGoal::haveDerivation()
buildResult.builtOutputs = assertPathValidity();
}
for (auto it = buildResult.builtOutputs.begin(); it != buildResult.builtOutputs.end(); ) {
for (auto it = buildResult.builtOutputs.begin(); it != buildResult.builtOutputs.end();) {
if (it->first != wantedOutput) {
it = buildResult.builtOutputs.erase(it);
} else {
@ -124,20 +124,20 @@ Goal::Co DerivationGoal::haveDerivation()
{
bool impure = drv->type().isImpure();
if (impure) experimentalFeatureSettings.require(Xp::ImpureDerivations);
if (impure)
experimentalFeatureSettings.require(Xp::ImpureDerivations);
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
for (auto & [outputName, outputHash] : outputHashes) {
InitialOutput v{
.wanted = true, // Will be refined later
.outputHash = outputHash
};
.outputHash = outputHash};
/* TODO we might want to also allow randomizing the paths
for regular CA derivations, e.g. for sake of checking
determinism. */
if (impure) {
v.known = InitialOutputStatus {
v.known = InitialOutputStatus{
.path = StorePath::random(outputPathName(drv->name, outputName)),
.status = PathStatus::Absent,
};
@ -173,22 +173,17 @@ Goal::Co DerivationGoal::haveDerivation()
them. */
if (settings.useSubstitutes && drvOptions.substitutesAllowed())
for (auto & [outputName, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.wanted)
continue;
if (!status.known)
waitees.insert(
upcast_goal(
worker.makeDrvOutputSubstitutionGoal(
DrvOutput{status.outputHash, outputName},
buildMode == bmRepair ? Repair : NoRepair
)
)
);
waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(
DrvOutput{status.outputHash, outputName}, buildMode == bmRepair ? Repair : NoRepair)));
else {
auto * cap = getDerivationCA(*drv);
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
status.known->path,
buildMode == bmRepair ? Repair : NoRepair,
cap ? std::optional { *cap } : std::nullopt)));
cap ? std::optional{*cap} : std::nullopt)));
}
}
@ -199,8 +194,11 @@ Goal::Co DerivationGoal::haveDerivation()
assert(!drv->type().isImpure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) {
co_return done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
co_return done(
BuildResult::TransientFailure,
{},
Error(
"some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
worker.store.printStorePath(drvPath)));
}
@ -215,26 +213,25 @@ Goal::Co DerivationGoal::haveDerivation()
co_return repairClosure();
}
if (buildMode == bmCheck && !allValid)
throw Error("some outputs of '%s' are not valid, so checking is not possible",
worker.store.printStorePath(drvPath));
throw Error(
"some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath));
/* Nothing to wait for; tail call */
co_return gaveUpOnSubstitution();
}
/**
* Used for `inputGoals` local variable below
*/
struct value_comparison
{
template <typename T>
bool operator()(const ref<T> & lhs, const ref<T> & rhs) const {
template<typename T>
bool operator()(const ref<T> & lhs, const ref<T> & rhs) const
{
return *lhs < *rhs;
}
};
Goal::Co DerivationGoal::repairClosure()
{
assert(!drv->type().isImpure());
@ -278,18 +275,20 @@ Goal::Co DerivationGoal::repairClosure()
/* Check each path (slow!). */
for (auto & i : outputClosure) {
if (worker.pathContentsGood(i)) continue;
if (worker.pathContentsGood(i))
continue;
printError(
"found corrupted or missing path '%s' in the output closure of '%s'",
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
worker.store.printStorePath(i),
worker.store.printStorePath(drvPath));
auto drvPath2 = outputsToDrv.find(i);
if (drvPath2 == outputsToDrv.end())
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else
waitees.insert(worker.makeGoal(
DerivedPath::Built {
DerivedPath::Built{
.drvPath = makeConstantStorePathRef(drvPath2->second),
.outputs = OutputsSpec::All { },
.outputs = OutputsSpec::All{},
},
bmRepair));
}
@ -299,18 +298,18 @@ Goal::Co DerivationGoal::repairClosure()
if (!waitees.empty()) {
trace("closure repaired");
if (nrFailed > 0)
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
throw Error(
"some paths in the output closure of derivation '%s' could not be repaired",
worker.store.printStorePath(drvPath));
}
co_return done(BuildResult::AlreadyValid, assertPathValidity());
}
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{
assert(!drv->type().isImpure());
for (auto * drvStore : { &worker.evalStore, &worker.store })
for (auto * drvStore : {&worker.evalStore, &worker.store})
if (drvStore->isValidPath(drvPath))
return worker.store.queryPartialDerivationOutputMap(drvPath, drvStore);
@ -326,7 +325,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
{
assert(!drv->type().isImpure());
for (auto * drvStore : { &worker.evalStore, &worker.store })
for (auto * drvStore : {&worker.evalStore, &worker.store})
if (drvStore->isValidPath(drvPath))
return worker.store.queryDerivationOutputMap(drvPath, drvStore);
@ -337,10 +336,10 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
return res;
}
std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
{
if (drv->type().isImpure()) return { false, {} };
if (drv->type().isImpure())
return {false, {}};
bool checkHash = buildMode == bmRepair;
StringSet wantedOutputsLeft{wantedOutput};
@ -359,11 +358,9 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
auto outputPath = *i.second;
info.known = {
.path = outputPath,
.status = !worker.store.isValidPath(outputPath)
? PathStatus::Absent
: !checkHash || worker.pathContentsGood(outputPath)
? PathStatus::Valid
: PathStatus::Corrupt,
.status = !worker.store.isValidPath(outputPath) ? PathStatus::Absent
: !checkHash || worker.pathContentsGood(outputPath) ? PathStatus::Valid
: PathStatus::Corrupt,
};
}
auto drvOutput = DrvOutput{info.outputHash, i.first};
@ -379,38 +376,38 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
// its realisation stored (probably because it has been built
// without the `ca-derivations` experimental flag).
worker.store.registerDrvOutput(
Realisation {
Realisation{
drvOutput,
info.known->path,
}
);
});
}
}
if (info.known && info.known->isValid())
validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
validOutputs.emplace(i.first, Realisation{drvOutput, info.known->path});
}
// If we requested all the outputs, we are always fine.
// If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid.
if (!wantedOutputsLeft.empty())
throw Error("derivation '%s' does not have wanted outputs %s",
throw Error(
"derivation '%s' does not have wanted outputs %s",
worker.store.printStorePath(drvPath),
concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
bool allValid = true;
for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.wanted)
continue;
if (!status.known || !status.known->isValid()) {
allValid = false;
break;
}
}
return { allValid, validOutputs };
return {allValid, validOutputs};
}
SingleDrvOutputs DerivationGoal::assertPathValidity()
{
auto [allValid, validOutputs] = checkPathValidity();
@ -419,11 +416,7 @@ SingleDrvOutputs DerivationGoal::assertPathValidity()
return validOutputs;
}
Goal::Done DerivationGoal::done(
BuildResult::Status status,
SingleDrvOutputs builtOutputs,
std::optional<Error> ex)
Goal::Done DerivationGoal::done(BuildResult::Status status, SingleDrvOutputs builtOutputs, std::optional<Error> ex)
{
buildResult.status = status;
if (ex)
@ -458,4 +451,4 @@ Goal::Done DerivationGoal::done(
return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex));
}
}
} // namespace nix

2
src/libstore/build/derivation-trampoline-goal.cc
View file

@ -172,4 +172,4 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation
co_return amDone(g->exitCode, g->ex);
}
}
} // namespace nix

51
src/libstore/build/drv-output-substitution-goal.cc
View file

@ -8,10 +8,7 @@
namespace nix {
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
const DrvOutput & id,
Worker & worker,
RepairFlag repair,
std::optional<ContentAddress> ca)
const DrvOutput & id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker, init())
, id(id)
{
@ -19,7 +16,6 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
trace("created");
}
Goal::Co DrvOutputSubstitutionGoal::init()
{
trace("init");
@ -40,32 +36,35 @@ Goal::Co DrvOutputSubstitutionGoal::init()
some other error occurs), so it must not touch `this`. So put
the shared state in a separate refcounted object. */
auto outPipe = std::make_shared<MuxablePipe>();
#ifndef _WIN32
#ifndef _WIN32
outPipe->create();
#else
#else
outPipe->createAsyncPipe(worker.ioport.get());
#endif
#endif
auto promise = std::make_shared<std::promise<std::shared_ptr<const Realisation>>>();
sub->queryRealisation(
id,
{ [outPipe(outPipe), promise(promise)](std::future<std::shared_ptr<const Realisation>> res) {
id, {[outPipe(outPipe), promise(promise)](std::future<std::shared_ptr<const Realisation>> res) {
try {
Finally updateStats([&]() { outPipe->writeSide.close(); });
promise->set_value(res.get());
} catch (...) {
promise->set_exception(std::current_exception());
}
} });
}});
worker.childStarted(shared_from_this(), {
#ifndef _WIN32
outPipe->readSide.get()
#else
&*outPipe
#endif
}, true, false);
worker.childStarted(
shared_from_this(),
{
#ifndef _WIN32
outPipe->readSide.get()
#else
&*outPipe
#endif
},
true,
false);
co_await Suspend{};
@ -84,7 +83,8 @@ Goal::Co DrvOutputSubstitutionGoal::init()
substituterFailed = true;
}
if (!outputInfo) continue;
if (!outputInfo)
continue;
bool failed = false;
@ -101,8 +101,7 @@ Goal::Co DrvOutputSubstitutionGoal::init()
sub->getUri(),
depId.to_string(),
worker.store.printStorePath(localOutputInfo->outPath),
worker.store.printStorePath(depPath)
);
worker.store.printStorePath(depPath));
failed = true;
break;
}
@ -110,7 +109,8 @@ Goal::Co DrvOutputSubstitutionGoal::init()
}
}
if (failed) continue;
if (failed)
continue;
co_return realisationFetched(std::move(waitees), outputInfo, sub);
}
@ -130,7 +130,9 @@ Goal::Co DrvOutputSubstitutionGoal::init()
co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters);
}
Goal::Co DrvOutputSubstitutionGoal::realisationFetched(Goals waitees, std::shared_ptr<const Realisation> outputInfo, nix::ref<nix::Store> sub) {
Goal::Co DrvOutputSubstitutionGoal::realisationFetched(
Goals waitees, std::shared_ptr<const Realisation> outputInfo, nix::ref<nix::Store> sub)
{
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
co_await await(std::move(waitees));
@ -160,5 +162,4 @@ void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)
worker.wakeUp(shared_from_this());
}
}
} // namespace nix

45
src/libstore/build/entry-points.cc
View file

@ -38,15 +38,14 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
ex->withExitStatus(worker.failingExitStatus());
throw std::move(*ex);
} else if (!failed.empty()) {
if (ex) logError(ex->info());
if (ex)
logError(ex->info());
throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
}
}
std::vector<KeyedBuildResult> Store::buildPathsWithResults(
const std::vector<DerivedPath> & reqs,
BuildMode buildMode,
std::shared_ptr<Store> evalStore)
const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
Worker worker(*this, evalStore ? *evalStore : *this);
@ -65,36 +64,36 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
results.reserve(state.size());
for (auto & [req, goalPtr] : state)
results.emplace_back(KeyedBuildResult {
goalPtr->buildResult,
/* .path = */ req,
});
results.emplace_back(
KeyedBuildResult{
goalPtr->buildResult,
/* .path = */ req,
});
return results;
}
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode)
{
Worker worker(*this, *this);
auto goal = worker.makeDerivationTrampolineGoal(drvPath, OutputsSpec::All {}, drv, buildMode);
auto goal = worker.makeDerivationTrampolineGoal(drvPath, OutputsSpec::All{}, drv, buildMode);
try {
worker.run(Goals{goal});
return goal->buildResult;
} catch (Error & e) {
return BuildResult {
return BuildResult{
.status = BuildResult::MiscFailure,
.errorMsg = e.msg(),
};
};
}
void Store::ensurePath(const StorePath & path)
{
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
if (isValidPath(path))
return;
Worker worker(*this, *this);
GoalPtr goal = worker.makePathSubstitutionGoal(path);
@ -107,11 +106,11 @@ void Store::ensurePath(const StorePath & path)
goal->ex->withExitStatus(worker.failingExitStatus());
throw std::move(*goal->ex);
} else
throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
throw Error(
worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
}
}
void Store::repairPath(const StorePath & path)
{
Worker worker(*this, *this);
@ -126,15 +125,17 @@ void Store::repairPath(const StorePath & path)
auto info = queryPathInfo(path);
if (info->deriver && isValidPath(*info->deriver)) {
goals.clear();
goals.insert(worker.makeGoal(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(*info->deriver),
// FIXME: Should just build the specific output we need.
.outputs = OutputsSpec::All { },
}, bmRepair));
goals.insert(worker.makeGoal(
DerivedPath::Built{
.drvPath = makeConstantStorePathRef(*info->deriver),
// FIXME: Should just build the specific output we need.
.outputs = OutputsSpec::All{},
},
bmRepair));
worker.run(goals);
} else
throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));
}
}
}
} // namespace nix

58
src/libstore/build/goal.cc
View file

@ -8,28 +8,35 @@ using promise_type = nix::Goal::promise_type;
using handle_type = nix::Goal::handle_type;
using Suspend = nix::Goal::Suspend;
Co::Co(Co&& rhs) {
Co::Co(Co && rhs)
{
this->handle = rhs.handle;
rhs.handle = nullptr;
}
void Co::operator=(Co&& rhs) {
void Co::operator=(Co && rhs)
{
this->handle = rhs.handle;
rhs.handle = nullptr;
}
Co::~Co() {
Co::~Co()
{
if (handle) {
handle.promise().alive = false;
handle.destroy();
}
}
Co promise_type::get_return_object() {
Co promise_type::get_return_object()
{
auto handle = handle_type::from_promise(*this);
return Co{handle};
};
std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h) noexcept {
auto& p = h.promise();
std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h) noexcept
{
auto & p = h.promise();
auto goal = p.goal;
assert(goal);
goal->trace("in final_awaiter");
@ -39,9 +46,9 @@ std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h
// We still have a continuation, i.e. work to do.
// We assert that the goal is still busy.
assert(goal->exitCode == ecBusy);
assert(goal->top_co); // Goal must have an active coroutine.
assert(goal->top_co); // Goal must have an active coroutine.
assert(goal->top_co->handle == h); // The active coroutine must be us.
assert(p.alive); // We must not have been destructed.
assert(p.alive); // We must not have been destructed.
// we move continuation to the top,
// note: previous top_co is actually h, so by moving into it,
@ -68,7 +75,8 @@ std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h
}
}
void promise_type::return_value(Co&& next) {
void promise_type::return_value(Co && next)
{
goal->trace("return_value(Co&&)");
// Save old continuation.
auto old_continuation = std::move(continuation);
@ -82,20 +90,22 @@ void promise_type::return_value(Co&& next) {
continuation->handle.promise().continuation = std::move(old_continuation);
}
std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) {
std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller)
{
assert(handle); // we must be a valid coroutine
auto& p = handle.promise();
auto & p = handle.promise();
assert(!p.continuation); // we must have no continuation
assert(!p.goal); // we must not have a goal yet
assert(!p.goal); // we must not have a goal yet
auto goal = caller.promise().goal;
assert(goal);
p.goal = goal;
p.continuation = std::move(goal->top_co); // we set our continuation to be top_co (i.e. caller)
goal->top_co = std::move(*this); // we set top_co to ourselves, don't use this anymore after this!
return p.goal->top_co->handle; // we execute ourselves
goal->top_co = std::move(*this); // we set top_co to ourselves, don't use this anymore after this!
return p.goal->top_co->handle; // we execute ourselves
}
bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
bool CompareGoalPtrs::operator()(const GoalPtr & a, const GoalPtr & b) const
{
std::string s1 = a->key();
std::string s2 = b->key();
return s1 < s2;
@ -146,9 +156,11 @@ Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
goal->trace(fmt("waitee '%s' done; %d left", name, goal->waitees.size()));
if (result == ecFailed || result == ecNoSubstituters) ++goal->nrFailed;
if (result == ecFailed || result == ecNoSubstituters)
++goal->nrFailed;
if (result == ecNoSubstituters) ++goal->nrNoSubstituters;
if (result == ecNoSubstituters)
++goal->nrNoSubstituters;
if (goal->waitees.empty()) {
worker.wakeUp(goal);
@ -177,7 +189,6 @@ Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
return Done{};
}
void Goal::trace(std::string_view s)
{
debug("%1%: %2%", name, s);
@ -194,22 +205,25 @@ void Goal::work()
assert(top_co || exitCode != ecBusy);
}
Goal::Co Goal::yield() {
Goal::Co Goal::yield()
{
worker.wakeUp(shared_from_this());
co_await Suspend{};
co_return Return{};
}
Goal::Co Goal::waitForAWhile() {
Goal::Co Goal::waitForAWhile()
{
worker.waitForAWhile(shared_from_this());
co_await Suspend{};
co_return Return{};
}
Goal::Co Goal::waitForBuildSlot() {
Goal::Co Goal::waitForBuildSlot()
{
worker.waitForBuildSlot(shared_from_this());
co_await Suspend{};
co_return Return{};
}
}
} // namespace nix

81
src/libstore/build/substitution-goal.cc
View file

@ -8,7 +8,8 @@
namespace nix {
PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
PathSubstitutionGoal::PathSubstitutionGoal(
const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker, init())
, storePath(storePath)
, repair(repair)
@ -19,17 +20,12 @@ PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker &
maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
}
PathSubstitutionGoal::~PathSubstitutionGoal()
{
cleanup();
}
Goal::Done PathSubstitutionGoal::done(
ExitCode result,
BuildResult::Status status,
std::optional<std::string> errorMsg)
Goal::Done PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status, std::optional<std::string> errorMsg)
{
buildResult.status = status;
if (errorMsg) {
@ -39,7 +35,6 @@ Goal::Done PathSubstitutionGoal::done(
return amDone(result);
}
Goal::Co PathSubstitutionGoal::init()
{
trace("init");
@ -52,7 +47,8 @@ Goal::Co PathSubstitutionGoal::init()
}
if (settings.readOnlyMode)
throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
throw Error(
"cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
@ -72,8 +68,7 @@ Goal::Co PathSubstitutionGoal::init()
if (ca) {
subPath = sub->makeFixedOutputPathFromCA(
std::string { storePath.name() },
ContentAddressWithReferences::withoutRefs(*ca));
std::string{storePath.name()}, ContentAddressWithReferences::withoutRefs(*ca));
if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) {
@ -86,13 +81,16 @@ Goal::Co PathSubstitutionGoal::init()
} catch (InvalidPath &) {
continue;
} catch (SubstituterDisabled & e) {
if (settings.tryFallback) continue;
else throw e;
if (settings.tryFallback)
continue;
else
throw e;
} catch (Error & e) {
if (settings.tryFallback) {
logError(e.info());
continue;
} else throw e;
} else
throw e;
}
if (info->path != storePath) {
@ -101,8 +99,11 @@ Goal::Co PathSubstitutionGoal::init()
info2->path = storePath;
info = info2;
} else {
printError("asked '%s' for '%s' but got '%s'",
sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
printError(
"asked '%s' for '%s' but got '%s'",
sub->getUri(),
worker.store.printStorePath(storePath),
sub->printStorePath(info->path));
continue;
}
}
@ -114,18 +115,19 @@ Goal::Co PathSubstitutionGoal::init()
maintainExpectedDownload =
narInfo && narInfo->fileSize
? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
: nullptr;
? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
: nullptr;
worker.updateProgress();
/* Bail out early if this substituter lacks a valid
signature. LocalStore::addToStore() also checks for this, but
only after we've downloaded the path. */
if (!sub->config.isTrusted && worker.store.pathInfoIsUntrusted(*info))
{
warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'",
worker.store.printStorePath(storePath), sub->getUri());
if (!sub->config.isTrusted && worker.store.pathInfoIsUntrusted(*info)) {
warn(
"ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'",
worker.store.printStorePath(storePath),
sub->getUri());
continue;
}
@ -159,11 +161,12 @@ Goal::Co PathSubstitutionGoal::init()
co_return done(
substituterFailed ? ecFailed : ecNoSubstituters,
BuildResult::NoSubstituters,
fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)));
fmt("path '%s' is required, but there is no substituter that can build it",
worker.store.printStorePath(storePath)));
}
Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed)
Goal::Co PathSubstitutionGoal::tryToRun(
StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed)
{
trace("all references realised");
@ -175,11 +178,13 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
}
for (auto & i : info->references)
/* ignore self-references */
/* ignore self-references */
if (i != storePath) {
if (!worker.store.isValidPath(i)) {
throw Error("reference '%s' of path '%s' is not a valid path",
worker.store.printStorePath(i), worker.store.printStorePath(storePath));
throw Error(
"reference '%s' of path '%s' is not a valid path",
worker.store.printStorePath(i),
worker.store.printStorePath(storePath));
}
}
@ -215,8 +220,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
PushActivity pact(act.id);
copyStorePath(*sub, worker.store,
subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs);
copyStorePath(*sub, worker.store, subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
} catch (...) {
@ -224,13 +228,17 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
}
});
worker.childStarted(shared_from_this(), {
worker.childStarted(
shared_from_this(),
{
#ifndef _WIN32
outPipe.readSide.get()
outPipe.readSide.get()
#else
&outPipe
&outPipe
#endif
}, true, false);
},
true,
false);
co_await Suspend{};
@ -282,13 +290,11 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
co_return done(ecSuccess, BuildResult::Substituted);
}
void PathSubstitutionGoal::handleEOF(Descriptor fd)
{
worker.wakeUp(shared_from_this());
}
void PathSubstitutionGoal::cleanup()
{
try {
@ -304,5 +310,4 @@ void PathSubstitutionGoal::cleanup()
}
}
}
} // namespace nix

202
src/libstore/build/worker.cc
View file

@ -29,7 +29,6 @@ Worker::Worker(Store & store, Store & evalStore)
checkMismatch = false;
}
Worker::~Worker()
{
/* Explicitly get rid of all strong pointers now. After this all
@ -44,9 +43,10 @@ Worker::~Worker()
}
template<class G, typename... Args>
std::shared_ptr<G> Worker::initGoalIfNeeded(std::weak_ptr<G> & goal_weak, Args && ...args)
std::shared_ptr<G> Worker::initGoalIfNeeded(std::weak_ptr<G> & goal_weak, Args &&... args)
{
if (auto goal = goal_weak.lock()) return goal;
if (auto goal = goal_weak.lock())
return goal;
auto goal = std::make_shared<G>(args...);
goal_weak = goal;
@ -55,64 +55,60 @@ std::shared_ptr<G> Worker::initGoalIfNeeded(std::weak_ptr<G> & goal_weak, Args &
}
std::shared_ptr<DerivationTrampolineGoal> Worker::makeDerivationTrampolineGoal(
ref<const SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs,
BuildMode buildMode)
ref<const SingleDerivedPath> drvReq, const OutputsSpec & wantedOutputs, BuildMode buildMode)
{
return initGoalIfNeeded(
derivationTrampolineGoals.ensureSlot(*drvReq).value[wantedOutputs],
drvReq, wantedOutputs, *this, buildMode);
derivationTrampolineGoals.ensureSlot(*drvReq).value[wantedOutputs], drvReq, wantedOutputs, *this, buildMode);
}
std::shared_ptr<DerivationTrampolineGoal> Worker::makeDerivationTrampolineGoal(
const StorePath & drvPath,
const OutputsSpec & wantedOutputs,
const Derivation & drv,
BuildMode buildMode)
const StorePath & drvPath, const OutputsSpec & wantedOutputs, const Derivation & drv, BuildMode buildMode)
{
return initGoalIfNeeded(
derivationTrampolineGoals.ensureSlot(DerivedPath::Opaque{drvPath}).value[wantedOutputs],
drvPath, wantedOutputs, drv, *this, buildMode);
drvPath,
wantedOutputs,
drv,
*this,
buildMode);
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
const Derivation & drv, const OutputName & wantedOutput, BuildMode buildMode)
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(
const StorePath & drvPath, const Derivation & drv, const OutputName & wantedOutput, BuildMode buildMode)
{
return initGoalIfNeeded(derivationGoals[drvPath][wantedOutput], drvPath, drv, wantedOutput, *this, buildMode);
}
std::shared_ptr<DerivationBuildingGoal> Worker::makeDerivationBuildingGoal(const StorePath & drvPath,
const Derivation & drv, BuildMode buildMode)
std::shared_ptr<DerivationBuildingGoal>
Worker::makeDerivationBuildingGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode)
{
return initGoalIfNeeded(derivationBuildingGoals[drvPath], drvPath, drv, *this, buildMode);
}
std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
std::shared_ptr<PathSubstitutionGoal>
Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
return initGoalIfNeeded(substitutionGoals[path], path, *this, repair, ca);
}
std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
std::shared_ptr<DrvOutputSubstitutionGoal>
Worker::makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair, std::optional<ContentAddress> ca)
{
return initGoalIfNeeded(drvOutputSubstitutionGoals[id], id, *this, repair, ca);
}
GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr {
return makeDerivationTrampolineGoal(bfd.drvPath, bfd.outputs, buildMode);
return std::visit(
overloaded{
[&](const DerivedPath::Built & bfd) -> GoalPtr {
return makeDerivationTrampolineGoal(bfd.drvPath, bfd.outputs, buildMode);
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
},
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
},
}, req.raw());
req.raw());
}
/**
@ -149,12 +145,12 @@ static bool removeGoal(std::shared_ptr<G> goal, std::map<K, Inner> & goalMap)
}
template<typename G>
static bool removeGoal(std::shared_ptr<G> goal, typename DerivedPathMap<std::map<OutputsSpec, std::weak_ptr<G>>>::ChildNode & node)
static bool
removeGoal(std::shared_ptr<G> goal, typename DerivedPathMap<std::map<OutputsSpec, std::weak_ptr<G>>>::ChildNode & node)
{
return removeGoal(goal, node.value) || removeGoal(goal, node.childMap);
}
void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationTrampolineGoal>(goal))
@ -181,34 +177,31 @@ void Worker::removeGoal(GoalPtr goal)
/* Wake up goals waiting for any goal to finish. */
for (auto & i : waitingForAnyGoal) {
GoalPtr goal = i.lock();
if (goal) wakeUp(goal);
if (goal)
wakeUp(goal);
}
waitingForAnyGoal.clear();
}
void Worker::wakeUp(GoalPtr goal)
{
goal->trace("woken up");
addToWeakGoals(awake, goal);
}
size_t Worker::getNrLocalBuilds()
{
return nrLocalBuilds;
}
size_t Worker::getNrSubstitutions()
{
return nrSubstitutions;
}
void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels,
bool inBuildSlot, bool respectTimeouts)
void Worker::childStarted(
GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels, bool inBuildSlot, bool respectTimeouts)
{
Child child;
child.goal = goal;
@ -235,12 +228,11 @@ void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::Com
}
}
void Worker::childTerminated(Goal * goal, bool wakeSleepers)
{
auto i = std::find_if(children.begin(), children.end(),
[&](const Child & child) { return child.goal2 == goal; });
if (i == children.end()) return;
auto i = std::find_if(children.begin(), children.end(), [&](const Child & child) { return child.goal2 == goal; });
if (i == children.end())
return;
if (i->inBuildSlot) {
switch (goal->jobCategory()) {
@ -267,40 +259,37 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
/* Wake up goals waiting for a build slot. */
for (auto & j : wantingToBuild) {
GoalPtr goal = j.lock();
if (goal) wakeUp(goal);
if (goal)
wakeUp(goal);
}
wantingToBuild.clear();
}
}
void Worker::waitForBuildSlot(GoalPtr goal)
{
goal->trace("wait for build slot");
bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution;
if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) ||
(isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))
if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs)
|| (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))
wakeUp(goal); /* we can do it right away */
else
addToWeakGoals(wantingToBuild, goal);
}
void Worker::waitForAnyGoal(GoalPtr goal)
{
debug("wait for any goal");
addToWeakGoals(waitingForAnyGoal, goal);
}
void Worker::waitForAWhile(GoalPtr goal)
{
debug("wait for a while");
addToWeakGoals(waitingForAWhile, goal);
}
void Worker::run(const Goals & _topGoals)
{
std::vector<nix::DerivedPath> topPaths;
@ -308,10 +297,11 @@ void Worker::run(const Goals & _topGoals)
for (auto & i : _topGoals) {
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationTrampolineGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built {
.drvPath = goal->drvReq,
.outputs = goal->wantedOutputs,
});
topPaths.push_back(
DerivedPath::Built{
.drvPath = goal->drvReq,
.outputs = goal->wantedOutputs,
});
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Opaque{goal->storePath});
}
@ -336,33 +326,37 @@ void Worker::run(const Goals & _topGoals)
Goals awake2;
for (auto & i : awake) {
GoalPtr goal = i.lock();
if (goal) awake2.insert(goal);
if (goal)
awake2.insert(goal);
}
awake.clear();
for (auto & goal : awake2) {
checkInterrupt();
goal->work();
if (topGoals.empty()) break; // stuff may have been cancelled
if (topGoals.empty())
break; // stuff may have been cancelled
}
}
if (topGoals.empty()) break;
if (topGoals.empty())
break;
/* Wait for input. */
if (!children.empty() || !waitingForAWhile.empty())
waitForInput();
else if (awake.empty() && 0U == settings.maxBuildJobs) {
if (getMachines().empty())
throw Error(
"Unable to start any build; either increase '--max-jobs' or enable remote builds.\n"
"\n"
"For more information run 'man nix.conf' and search for '/machines'.");
throw Error(
"Unable to start any build; either increase '--max-jobs' or enable remote builds.\n"
"\n"
"For more information run 'man nix.conf' and search for '/machines'.");
else
throw Error(
"Unable to start any build; remote machines may not have all required system features.\n"
"\n"
"For more information run 'man nix.conf' and search for '/machines'.");
} else assert(!awake.empty());
throw Error(
"Unable to start any build; remote machines may not have all required system features.\n"
"\n"
"For more information run 'man nix.conf' and search for '/machines'.");
} else
assert(!awake.empty());
}
/* If --keep-going is not set, it's possible that the main goal
@ -395,7 +389,8 @@ void Worker::waitForInput()
// Periodicallty wake up to see if we need to run the garbage collector.
nearest = before + std::chrono::seconds(10);
for (auto & i : children) {
if (!i.respectTimeouts) continue;
if (!i.respectTimeouts)
continue;
if (0 != settings.maxSilentTime)
nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
if (0 != settings.buildTimeout)
@ -410,11 +405,15 @@ void Worker::waitForInput()
up after a few seconds at most. */
if (!waitingForAWhile.empty()) {
useTimeout = true;
if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
timeout = std::max(1L,
if (lastWokenUp == steady_time_point::min() || lastWokenUp > before)
lastWokenUp = before;
timeout = std::max(
1L,
(long) std::chrono::duration_cast<std::chrono::seconds>(
lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
} else lastWokenUp = steady_time_point::min();
lastWokenUp + std::chrono::seconds(settings.pollInterval) - before)
.count());
} else
lastWokenUp = steady_time_point::min();
if (useTimeout)
vomit("sleeping %d seconds", timeout);
@ -427,7 +426,7 @@ void Worker::waitForInput()
includes EOF. */
for (auto & i : children) {
for (auto & j : i.channels) {
state.pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
state.pollStatus.push_back((struct pollfd) {.fd = j, .events = POLLIN});
state.fdToPollStatus[j] = state.pollStatus.size() - 1;
}
}
@ -437,7 +436,7 @@ void Worker::waitForInput()
#ifdef _WIN32
ioport.get(),
#endif
useTimeout ? (std::optional { timeout * 1000 }) : std::nullopt);
useTimeout ? (std::optional{timeout * 1000}) : std::nullopt);
auto after = steady_time_point::clock::now();
@ -455,8 +454,7 @@ void Worker::waitForInput()
state.iterate(
j->channels,
[&](Descriptor k, std::string_view data) {
printMsg(lvlVomit, "%1%: read %2% bytes",
goal->getName(), data.size());
printMsg(lvlVomit, "%1%: read %2% bytes", goal->getName(), data.size());
j->lastOutput = after;
goal->handleChildOutput(k, data);
},
@ -465,24 +463,16 @@ void Worker::waitForInput()
goal->handleEOF(k);
});
if (goal->exitCode == Goal::ecBusy &&
0 != settings.maxSilentTime &&
j->respectTimeouts &&
after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
{
goal->timedOut(Error(
"%1% timed out after %2% seconds of silence",
goal->getName(), settings.maxSilentTime));
if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts
&& after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) {
goal->timedOut(
Error("%1% timed out after %2% seconds of silence", goal->getName(), settings.maxSilentTime));
}
else if (goal->exitCode == Goal::ecBusy &&
0 != settings.buildTimeout &&
j->respectTimeouts &&
after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
{
goal->timedOut(Error(
"%1% timed out after %2% seconds",
goal->getName(), settings.buildTimeout));
else if (
goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts
&& after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) {
goal->timedOut(Error("%1% timed out after %2% seconds", goal->getName(), settings.buildTimeout));
}
}
@ -490,26 +480,26 @@ void Worker::waitForInput()
lastWokenUp = after;
for (auto & i : waitingForAWhile) {
GoalPtr goal = i.lock();
if (goal) wakeUp(goal);
if (goal)
wakeUp(goal);
}
waitingForAWhile.clear();
}
}
unsigned int Worker::failingExitStatus()
{
// See API docs in header for explanation
unsigned int mask = 0;
bool buildFailure = permanentFailure || timedOut || hashMismatch;
if (buildFailure)
mask |= 0x04; // 100
mask |= 0x04; // 100
if (timedOut)
mask |= 0x01; // 101
mask |= 0x01; // 101
if (hashMismatch)
mask |= 0x02; // 102
mask |= 0x02; // 102
if (checkMismatch) {
mask |= 0x08; // 104
mask |= 0x08; // 104
}
if (mask)
@ -517,11 +507,11 @@ unsigned int Worker::failingExitStatus()
return mask ? mask : 1;
}
bool Worker::pathContentsGood(const StorePath & path)
{
auto i = pathContentsGoodCache.find(path);
if (i != pathContentsGoodCache.end()) return i->second;
if (i != pathContentsGoodCache.end())
return i->second;
printInfo("checking path '%s'...", store.printStorePath(path));
auto info = store.queryPathInfo(path);
bool res;
@ -529,8 +519,10 @@ bool Worker::pathContentsGood(const StorePath & path)
res = false;
else {
auto current = hashPath(
{store.getFSAccessor(), CanonPath(path.to_string())},
FileIngestionMethod::NixArchive, info->narHash.algo).first;
{store.getFSAccessor(), CanonPath(path.to_string())},
FileIngestionMethod::NixArchive,
info->narHash.algo)
.first;
Hash nullHash(HashAlgorithm::SHA256);
res = info->narHash == nullHash || info->narHash == current;
}
@ -540,13 +532,11 @@ bool Worker::pathContentsGood(const StorePath & path)
return res;
}
void Worker::markContentsGood(const StorePath & path)
{
pathContentsGoodCache.insert_or_assign(path, true);
}
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal)
{
return subGoal;
@ -562,4 +552,4 @@ GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
return subGoal;
}
}
} // namespace nix