mirror of
https://github.com/NixOS/nix.git
synced 2025-12-16 22:11:05 +01:00
Merge remote-tracking branch 'upstream/master' into path-info
This commit is contained in:
John Ericson
commit
8ba089597f
434 changed files with 23391 additions and 36322 deletions
|
|
@ -17,6 +17,7 @@
|
|||
#include <regex>
|
||||
#include <queue>
|
||||
|
||||
#include <fstream>
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
|
|
@ -65,7 +66,7 @@ namespace nix {
|
|||
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker)
|
||||
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
|
||||
, useDerivation(true)
|
||||
, drvPath(drvPath)
|
||||
, wantedOutputs(wantedOutputs)
|
||||
|
|
@ -84,7 +85,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
|
|||
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker)
|
||||
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
|
||||
, useDerivation(false)
|
||||
, drvPath(drvPath)
|
||||
, wantedOutputs(wantedOutputs)
|
||||
|
|
@ -115,7 +116,7 @@ DerivationGoal::~DerivationGoal()
|
|||
}
|
||||
|
||||
|
||||
string DerivationGoal::key()
|
||||
std::string DerivationGoal::key()
|
||||
{
|
||||
/* Ensure that derivations get built in order of their name,
|
||||
i.e. a derivation named "aardvark" always comes before
|
||||
|
|
@ -134,7 +135,7 @@ void DerivationGoal::killChild()
|
|||
void DerivationGoal::timedOut(Error && ex)
|
||||
{
|
||||
killChild();
|
||||
done(BuildResult::TimedOut, ex);
|
||||
done(BuildResult::TimedOut, {}, ex);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -181,7 +182,7 @@ void DerivationGoal::loadDerivation()
|
|||
trace("loading derivation");
|
||||
|
||||
if (nrFailed != 0) {
|
||||
done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
|
||||
done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -193,7 +194,7 @@ void DerivationGoal::loadDerivation()
|
|||
assert(worker.evalStore.isValidPath(drvPath));
|
||||
|
||||
/* Get the derivation. */
|
||||
drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath));
|
||||
drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath));
|
||||
|
||||
haveDerivation();
|
||||
}
|
||||
|
|
@ -204,7 +205,7 @@ void DerivationGoal::haveDerivation()
|
|||
trace("have derivation");
|
||||
|
||||
if (drv->type() == DerivationType::CAFloating)
|
||||
settings.requireExperimentalFeature("ca-derivations");
|
||||
settings.requireExperimentalFeature(Xp::CaDerivations);
|
||||
|
||||
retrySubstitution = false;
|
||||
|
||||
|
|
@ -214,28 +215,20 @@ void DerivationGoal::haveDerivation()
|
|||
|
||||
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
|
||||
for (auto & [outputName, outputHash] : outputHashes)
|
||||
initialOutputs.insert({
|
||||
initialOutputs.insert({
|
||||
outputName,
|
||||
InitialOutput{
|
||||
InitialOutput {
|
||||
.wanted = true, // Will be refined later
|
||||
.outputHash = outputHash
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/* Check what outputs paths are not already valid. */
|
||||
checkPathValidity();
|
||||
bool allValid = true;
|
||||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
if (!status.known || !status.known->isValid()) {
|
||||
allValid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
/* If they are all valid, then we're done. */
|
||||
if (allValid && buildMode == bmNormal) {
|
||||
done(BuildResult::AlreadyValid);
|
||||
done(BuildResult::AlreadyValid, std::move(validOutputs));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -276,8 +269,8 @@ void DerivationGoal::outputsSubstitutionTried()
|
|||
trace("all outputs substituted (maybe)");
|
||||
|
||||
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
|
||||
done(BuildResult::TransientFailure,
|
||||
fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
|
||||
done(BuildResult::TransientFailure, {},
|
||||
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
|
||||
worker.store.printStorePath(drvPath)));
|
||||
return;
|
||||
}
|
||||
|
|
@ -300,23 +293,17 @@ void DerivationGoal::outputsSubstitutionTried()
|
|||
return;
|
||||
}
|
||||
|
||||
checkPathValidity();
|
||||
size_t nrInvalid = 0;
|
||||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
if (!status.known || !status.known->isValid())
|
||||
nrInvalid++;
|
||||
}
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
if (buildMode == bmNormal && nrInvalid == 0) {
|
||||
done(BuildResult::Substituted);
|
||||
if (buildMode == bmNormal && allValid) {
|
||||
done(BuildResult::Substituted, std::move(validOutputs));
|
||||
return;
|
||||
}
|
||||
if (buildMode == bmRepair && nrInvalid == 0) {
|
||||
if (buildMode == bmRepair && allValid) {
|
||||
repairClosure();
|
||||
return;
|
||||
}
|
||||
if (buildMode == bmCheck && nrInvalid > 0)
|
||||
if (buildMode == bmCheck && !allValid)
|
||||
throw Error("some outputs of '%s' are not valid, so checking is not possible",
|
||||
worker.store.printStorePath(drvPath));
|
||||
|
||||
|
|
@ -408,7 +395,7 @@ void DerivationGoal::repairClosure()
|
|||
}
|
||||
|
||||
if (waitees.empty()) {
|
||||
done(BuildResult::AlreadyValid);
|
||||
done(BuildResult::AlreadyValid, assertPathValidity());
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -422,7 +409,7 @@ void DerivationGoal::closureRepaired()
|
|||
if (nrFailed > 0)
|
||||
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
|
||||
worker.store.printStorePath(drvPath));
|
||||
done(BuildResult::AlreadyValid);
|
||||
done(BuildResult::AlreadyValid, assertPathValidity());
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -433,7 +420,7 @@ void DerivationGoal::inputsRealised()
|
|||
if (nrFailed != 0) {
|
||||
if (!useDerivation)
|
||||
throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
|
||||
done(BuildResult::DependencyFailed, Error(
|
||||
done(BuildResult::DependencyFailed, {}, Error(
|
||||
"%s dependencies of derivation '%s' failed to build",
|
||||
nrFailed, worker.store.printStorePath(drvPath)));
|
||||
return;
|
||||
|
|
@ -453,7 +440,7 @@ void DerivationGoal::inputsRealised()
|
|||
if (useDerivation) {
|
||||
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
|
||||
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
|
||||
((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
|
||||
|| fullDrv.type() == DerivationType::DeferredInputAddressed)) {
|
||||
/* We are be able to resolve this derivation based on the
|
||||
|
|
@ -464,7 +451,6 @@ void DerivationGoal::inputsRealised()
|
|||
Derivation drvResolved { *std::move(attempt) };
|
||||
|
||||
auto pathResolved = writeDerivation(worker.store, drvResolved);
|
||||
resolvedDrv = drvResolved;
|
||||
|
||||
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
|
||||
worker.store.printStorePath(drvPath),
|
||||
|
|
@ -475,9 +461,9 @@ void DerivationGoal::inputsRealised()
|
|||
worker.store.printStorePath(pathResolved),
|
||||
});
|
||||
|
||||
auto resolvedGoal = worker.makeDerivationGoal(
|
||||
resolvedDrvGoal = worker.makeDerivationGoal(
|
||||
pathResolved, wantedOutputs, buildMode);
|
||||
addWaitee(resolvedGoal);
|
||||
addWaitee(resolvedDrvGoal);
|
||||
|
||||
state = &DerivationGoal::resolvedFinished;
|
||||
return;
|
||||
|
|
@ -523,10 +509,11 @@ void DerivationGoal::inputsRealised()
|
|||
state = &DerivationGoal::tryToBuild;
|
||||
worker.wakeUp(shared_from_this());
|
||||
|
||||
result = BuildResult();
|
||||
buildResult = BuildResult { .path = buildResult.path };
|
||||
}
|
||||
|
||||
void DerivationGoal::started() {
|
||||
void DerivationGoal::started()
|
||||
{
|
||||
auto msg = fmt(
|
||||
buildMode == bmRepair ? "repairing outputs of '%s'" :
|
||||
buildMode == bmCheck ? "checking outputs of '%s'" :
|
||||
|
|
@ -588,19 +575,12 @@ void DerivationGoal::tryToBuild()
|
|||
omitted, but that would be less efficient.) Note that since we
|
||||
now hold the locks on the output paths, no other process can
|
||||
build this derivation, so no further checks are necessary. */
|
||||
checkPathValidity();
|
||||
bool allValid = true;
|
||||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
if (!status.known || !status.known->isValid()) {
|
||||
allValid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
if (buildMode != bmCheck && allValid) {
|
||||
debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
|
||||
outputLocks.setDeletion(true);
|
||||
done(BuildResult::AlreadyValid);
|
||||
done(BuildResult::AlreadyValid, std::move(validOutputs));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -616,7 +596,9 @@ void DerivationGoal::tryToBuild()
|
|||
/* Don't do a remote build if the derivation has the attribute
|
||||
`preferLocalBuild' set. Also, check and repair modes are only
|
||||
supported for local builds. */
|
||||
bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
|
||||
bool buildLocally =
|
||||
(buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store))
|
||||
&& settings.maxBuildJobs.get() != 0;
|
||||
|
||||
if (!buildLocally) {
|
||||
switch (tryBuildHook()) {
|
||||
|
|
@ -624,7 +606,7 @@ void DerivationGoal::tryToBuild()
|
|||
/* Yes, it has started doing so. Wait until we get
|
||||
EOF from the hook. */
|
||||
actLock.reset();
|
||||
result.startTime = time(0); // inexact
|
||||
buildResult.startTime = time(0); // inexact
|
||||
state = &DerivationGoal::buildDone;
|
||||
started();
|
||||
return;
|
||||
|
|
@ -653,7 +635,7 @@ void DerivationGoal::tryLocalBuild() {
|
|||
throw Error(
|
||||
"unable to build with a primary store that isn't a local store; "
|
||||
"either pass a different '--store' or enable remote builds."
|
||||
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
|
||||
"\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -828,8 +810,8 @@ void DerivationGoal::buildDone()
|
|||
|
||||
debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
|
||||
|
||||
result.timesBuilt++;
|
||||
result.stopTime = time(0);
|
||||
buildResult.timesBuilt++;
|
||||
buildResult.stopTime = time(0);
|
||||
|
||||
/* So the child is gone now. */
|
||||
worker.childTerminated(this);
|
||||
|
|
@ -874,11 +856,11 @@ void DerivationGoal::buildDone()
|
|||
|
||||
/* Compute the FS closure of the outputs and register them as
|
||||
being valid. */
|
||||
registerOutputs();
|
||||
auto builtOutputs = registerOutputs();
|
||||
|
||||
StorePathSet outputPaths;
|
||||
for (auto & [_, path] : finalOutputs)
|
||||
outputPaths.insert(path);
|
||||
for (auto & [_, output] : buildResult.builtOutputs)
|
||||
outputPaths.insert(output.outPath);
|
||||
runPostBuildHook(
|
||||
worker.store,
|
||||
*logger,
|
||||
|
|
@ -888,7 +870,7 @@ void DerivationGoal::buildDone()
|
|||
|
||||
if (buildMode == bmCheck) {
|
||||
cleanupPostOutputsRegisteredModeCheck();
|
||||
done(BuildResult::Built);
|
||||
done(BuildResult::Built, std::move(builtOutputs));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -909,6 +891,8 @@ void DerivationGoal::buildDone()
|
|||
outputLocks.setDeletion(true);
|
||||
outputLocks.unlock();
|
||||
|
||||
done(BuildResult::Built, std::move(builtOutputs));
|
||||
|
||||
} catch (BuildError & e) {
|
||||
outputLocks.unlock();
|
||||
|
||||
|
|
@ -928,30 +912,32 @@ void DerivationGoal::buildDone()
|
|||
BuildResult::PermanentFailure;
|
||||
}
|
||||
|
||||
done(st, e);
|
||||
done(st, {}, e);
|
||||
return;
|
||||
}
|
||||
|
||||
done(BuildResult::Built);
|
||||
}
|
||||
|
||||
void DerivationGoal::resolvedFinished() {
|
||||
assert(resolvedDrv);
|
||||
void DerivationGoal::resolvedFinished()
|
||||
{
|
||||
assert(resolvedDrvGoal);
|
||||
auto resolvedDrv = *resolvedDrvGoal->drv;
|
||||
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
|
||||
|
||||
StorePathSet outputPaths;
|
||||
|
||||
// `wantedOutputs` might be empty, which means “all the outputs”
|
||||
auto realWantedOutputs = wantedOutputs;
|
||||
if (realWantedOutputs.empty())
|
||||
realWantedOutputs = resolvedDrv->outputNames();
|
||||
realWantedOutputs = resolvedDrv.outputNames();
|
||||
|
||||
DrvOutputs builtOutputs;
|
||||
|
||||
for (auto & wantedOutput : realWantedOutputs) {
|
||||
assert(initialOutputs.count(wantedOutput) != 0);
|
||||
assert(resolvedHashes.count(wantedOutput) != 0);
|
||||
auto realisation = worker.store.queryRealisation(
|
||||
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
|
||||
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
|
||||
);
|
||||
// We've just built it, but maybe the build failed, in which case the
|
||||
// realisation won't be there
|
||||
|
|
@ -963,10 +949,11 @@ void DerivationGoal::resolvedFinished() {
|
|||
signRealisation(newRealisation);
|
||||
worker.store.registerDrvOutput(newRealisation);
|
||||
outputPaths.insert(realisation->outPath);
|
||||
builtOutputs.emplace(realisation->id, *realisation);
|
||||
} else {
|
||||
// If we don't have a realisation, then it must mean that something
|
||||
// failed when building the resolved drv
|
||||
assert(!result.success());
|
||||
assert(!buildResult.success());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -977,9 +964,17 @@ void DerivationGoal::resolvedFinished() {
|
|||
outputPaths
|
||||
);
|
||||
|
||||
// This is potentially a bit fishy in terms of error reporting. Not sure
|
||||
// how to do it in a cleaner way
|
||||
amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
|
||||
auto status = [&]() {
|
||||
auto & resolvedResult = resolvedDrvGoal->buildResult;
|
||||
switch (resolvedResult.status) {
|
||||
case BuildResult::AlreadyValid:
|
||||
return BuildResult::ResolvesToAlreadyValid;
|
||||
default:
|
||||
return resolvedResult.status;
|
||||
}
|
||||
}();
|
||||
|
||||
done(status, std::move(builtOutputs));
|
||||
}
|
||||
|
||||
HookReply DerivationGoal::tryBuildHook()
|
||||
|
|
@ -1002,7 +997,7 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
|
||||
/* Read the first line of input, which should be a word indicating
|
||||
whether the hook wishes to perform the build. */
|
||||
string reply;
|
||||
std::string reply;
|
||||
while (true) {
|
||||
auto s = [&]() {
|
||||
try {
|
||||
|
|
@ -1014,8 +1009,8 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
}();
|
||||
if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
|
||||
;
|
||||
else if (string(s, 0, 2) == "# ") {
|
||||
reply = string(s, 2);
|
||||
else if (s.substr(0, 2) == "# ") {
|
||||
reply = s.substr(2);
|
||||
break;
|
||||
}
|
||||
else {
|
||||
|
|
@ -1080,7 +1075,7 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
/* Create the log file and pipe. */
|
||||
Path logFile = openLogFile();
|
||||
|
||||
set<int> fds;
|
||||
std::set<int> fds;
|
||||
fds.insert(hook->fromHook.readSide.get());
|
||||
fds.insert(hook->builderOut.readSide.get());
|
||||
worker.childStarted(shared_from_this(), fds, false, false);
|
||||
|
|
@ -1089,7 +1084,7 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
}
|
||||
|
||||
|
||||
void DerivationGoal::registerOutputs()
|
||||
DrvOutputs DerivationGoal::registerOutputs()
|
||||
{
|
||||
/* When using a build hook, the build hook can register the output
|
||||
as valid (by doing `nix-store --import'). If so we don't have
|
||||
|
|
@ -1098,21 +1093,7 @@ void DerivationGoal::registerOutputs()
|
|||
We can only early return when the outputs are known a priori. For
|
||||
floating content-addressed derivations this isn't the case.
|
||||
*/
|
||||
for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
|
||||
if (!wantOutput(outputName, wantedOutputs))
|
||||
continue;
|
||||
if (!optOutputPath)
|
||||
throw BuildError(
|
||||
"output '%s' from derivation '%s' does not have a known output path",
|
||||
outputName, worker.store.printStorePath(drvPath));
|
||||
auto & outputPath = *optOutputPath;
|
||||
if (!worker.store.isValidPath(outputPath))
|
||||
throw BuildError(
|
||||
"output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid",
|
||||
outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath));
|
||||
|
||||
finalOutputs.insert_or_assign(outputName, outputPath);
|
||||
}
|
||||
return assertPathValidity();
|
||||
}
|
||||
|
||||
Path DerivationGoal::openLogFile()
|
||||
|
|
@ -1129,10 +1110,10 @@ Path DerivationGoal::openLogFile()
|
|||
logDir = localStore->logDir;
|
||||
else
|
||||
logDir = settings.nixLogDir;
|
||||
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2));
|
||||
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
|
||||
createDirs(dir);
|
||||
|
||||
Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
|
||||
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2),
|
||||
settings.compressLog ? ".bz2" : "");
|
||||
|
||||
fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
|
||||
|
|
@ -1164,16 +1145,17 @@ bool DerivationGoal::isReadDesc(int fd)
|
|||
return fd == hook->builderOut.readSide.get();
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::handleChildOutput(int fd, const string & data)
|
||||
void DerivationGoal::handleChildOutput(int fd, std::string_view data)
|
||||
{
|
||||
if (isReadDesc(fd))
|
||||
// local & `ssh://`-builds are dealt with here.
|
||||
auto isWrittenToLog = isReadDesc(fd);
|
||||
if (isWrittenToLog)
|
||||
{
|
||||
logSize += data.size();
|
||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||
killChild();
|
||||
done(
|
||||
BuildResult::LogLimitExceeded,
|
||||
BuildResult::LogLimitExceeded, {},
|
||||
Error("%s killed after writing more than %d bytes of log output",
|
||||
getName(), settings.maxLogSize));
|
||||
return;
|
||||
|
|
@ -1196,7 +1178,16 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
|
|||
if (hook && fd == hook->fromHook.readSide.get()) {
|
||||
for (auto c : data)
|
||||
if (c == '\n') {
|
||||
handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
|
||||
auto json = parseJSONMessage(currentHookLine);
|
||||
if (json) {
|
||||
auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
|
||||
// ensure that logs from a builder using `ssh-ng://` as protocol
|
||||
// are also available to `nix log`.
|
||||
if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) {
|
||||
auto f = (*json)["fields"];
|
||||
(*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n");
|
||||
}
|
||||
}
|
||||
currentHookLine.clear();
|
||||
} else
|
||||
currentHookLine += c;
|
||||
|
|
@ -1253,10 +1244,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
|||
}
|
||||
|
||||
|
||||
void DerivationGoal::checkPathValidity()
|
||||
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
|
||||
{
|
||||
bool checkHash = buildMode == bmRepair;
|
||||
auto wantedOutputsLeft = wantedOutputs;
|
||||
DrvOutputs validOutputs;
|
||||
|
||||
for (auto & i : queryPartialDerivationOutputMap()) {
|
||||
InitialOutput & info = initialOutputs.at(i.first);
|
||||
info.wanted = wantOutput(i.first, wantedOutputs);
|
||||
|
|
@ -1273,26 +1266,28 @@ void DerivationGoal::checkPathValidity()
|
|||
: PathStatus::Corrupt,
|
||||
};
|
||||
}
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
|
||||
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
|
||||
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
|
||||
if (auto real = worker.store.queryRealisation(drvOutput)) {
|
||||
info.known = {
|
||||
.path = real->outPath,
|
||||
.status = PathStatus::Valid,
|
||||
};
|
||||
} else if (info.known && info.known->status == PathStatus::Valid) {
|
||||
// We know the output because it' a static output of the
|
||||
} else if (info.known && info.known->isValid()) {
|
||||
// We know the output because it's a static output of the
|
||||
// derivation, and the output path is valid, but we don't have
|
||||
// its realisation stored (probably because it has been built
|
||||
// without the `ca-derivations` experimental flag)
|
||||
// without the `ca-derivations` experimental flag).
|
||||
worker.store.registerDrvOutput(
|
||||
Realisation{
|
||||
Realisation {
|
||||
drvOutput,
|
||||
info.known->path,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
if (info.wanted && info.known && info.known->isValid())
|
||||
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
|
||||
}
|
||||
// If we requested all the outputs via the empty set, we are always fine.
|
||||
// If we requested specific elements, the loop above removes all the valid
|
||||
|
|
@ -1301,24 +1296,50 @@ void DerivationGoal::checkPathValidity()
|
|||
throw Error("derivation '%s' does not have wanted outputs %s",
|
||||
worker.store.printStorePath(drvPath),
|
||||
concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
|
||||
|
||||
bool allValid = true;
|
||||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
if (!status.known || !status.known->isValid()) {
|
||||
allValid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { allValid, validOutputs };
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
|
||||
DrvOutputs DerivationGoal::assertPathValidity()
|
||||
{
|
||||
result.status = status;
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
if (!allValid)
|
||||
throw Error("some outputs are unexpectedly invalid");
|
||||
return validOutputs;
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::done(
|
||||
BuildResult::Status status,
|
||||
DrvOutputs builtOutputs,
|
||||
std::optional<Error> ex)
|
||||
{
|
||||
buildResult.status = status;
|
||||
if (ex)
|
||||
result.errorMsg = ex->what();
|
||||
amDone(result.success() ? ecSuccess : ecFailed, ex);
|
||||
if (result.status == BuildResult::TimedOut)
|
||||
// FIXME: strip: "error: "
|
||||
buildResult.errorMsg = ex->what();
|
||||
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
|
||||
if (buildResult.status == BuildResult::TimedOut)
|
||||
worker.timedOut = true;
|
||||
if (result.status == BuildResult::PermanentFailure)
|
||||
if (buildResult.status == BuildResult::PermanentFailure)
|
||||
worker.permanentFailure = true;
|
||||
|
||||
mcExpectedBuilds.reset();
|
||||
mcRunningBuilds.reset();
|
||||
|
||||
if (result.success()) {
|
||||
if (buildResult.success()) {
|
||||
assert(!builtOutputs.empty());
|
||||
buildResult.builtOutputs = std::move(builtOutputs);
|
||||
if (status == BuildResult::Built)
|
||||
worker.doneBuilds++;
|
||||
} else {
|
||||
|
|
@ -1327,6 +1348,13 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
|
|||
}
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
auto traceBuiltOutputsFile = getEnv("_NIX_TRACE_BUILT_OUTPUTS").value_or("");
|
||||
if (traceBuiltOutputsFile != "") {
|
||||
std::fstream fs;
|
||||
fs.open(traceBuiltOutputsFile, std::fstream::out);
|
||||
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -50,8 +50,8 @@ struct DerivationGoal : public Goal
|
|||
/* The path of the derivation. */
|
||||
StorePath drvPath;
|
||||
|
||||
/* The path of the corresponding resolved derivation */
|
||||
std::optional<BasicDerivation> resolvedDrv;
|
||||
/* The goal for the corresponding resolved derivation */
|
||||
std::shared_ptr<DerivationGoal> resolvedDrvGoal;
|
||||
|
||||
/* The specific outputs that we need to build. Empty means all of
|
||||
them. */
|
||||
|
|
@ -104,20 +104,8 @@ struct DerivationGoal : public Goal
|
|||
typedef void (DerivationGoal::*GoalState)();
|
||||
GoalState state;
|
||||
|
||||
/* The final output paths of the build.
|
||||
|
||||
- For input-addressed derivations, always the precomputed paths
|
||||
|
||||
- For content-addressed derivations, calcuated from whatever the hash
|
||||
ends up being. (Note that fixed outputs derivations that produce the
|
||||
"wrong" output still install that data under its true content-address.)
|
||||
*/
|
||||
OutputPathMap finalOutputs;
|
||||
|
||||
BuildMode buildMode;
|
||||
|
||||
BuildResult result;
|
||||
|
||||
/* The current round, if we're building multiple times. */
|
||||
size_t curRound = 1;
|
||||
|
||||
|
|
@ -145,15 +133,13 @@ struct DerivationGoal : public Goal
|
|||
|
||||
void timedOut(Error && ex) override;
|
||||
|
||||
string key() override;
|
||||
std::string key() override;
|
||||
|
||||
void work() override;
|
||||
|
||||
/* Add wanted outputs to an already existing derivation goal. */
|
||||
void addWantedOutputs(const StringSet & outputs);
|
||||
|
||||
BuildResult getResult() { return result; }
|
||||
|
||||
/* The states. */
|
||||
void getDerivation();
|
||||
void loadDerivation();
|
||||
|
|
@ -175,7 +161,7 @@ struct DerivationGoal : public Goal
|
|||
|
||||
/* Check that the derivation outputs all exist and register them
|
||||
as valid. */
|
||||
virtual void registerOutputs();
|
||||
virtual DrvOutputs registerOutputs();
|
||||
|
||||
/* Open a log file and a pipe to it. */
|
||||
Path openLogFile();
|
||||
|
|
@ -200,7 +186,7 @@ struct DerivationGoal : public Goal
|
|||
virtual bool isReadDesc(int fd);
|
||||
|
||||
/* Callback used by the worker to write to the log. */
|
||||
void handleChildOutput(int fd, const string & data) override;
|
||||
void handleChildOutput(int fd, std::string_view data) override;
|
||||
void handleEOF(int fd) override;
|
||||
void flushLine();
|
||||
|
||||
|
|
@ -210,8 +196,17 @@ struct DerivationGoal : public Goal
|
|||
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
|
||||
OutputPathMap queryDerivationOutputMap();
|
||||
|
||||
/* Return the set of (in)valid paths. */
|
||||
void checkPathValidity();
|
||||
/* Update 'initialOutputs' to determine the current status of the
|
||||
outputs of the derivation. Also returns a Boolean denoting
|
||||
whether all outputs are valid and non-corrupt, and a
|
||||
'DrvOutputs' structure containing the valid and wanted
|
||||
outputs. */
|
||||
std::pair<bool, DrvOutputs> checkPathValidity();
|
||||
|
||||
/* Aborts if any output is not valid or corrupt, and otherwise
|
||||
returns a 'DrvOutputs' structure containing the wanted
|
||||
outputs. */
|
||||
DrvOutputs assertPathValidity();
|
||||
|
||||
/* Forcibly kill the child process, if any. */
|
||||
virtual void killChild();
|
||||
|
|
@ -222,6 +217,7 @@ struct DerivationGoal : public Goal
|
|||
|
||||
void done(
|
||||
BuildResult::Status status,
|
||||
DrvOutputs builtOutputs = {},
|
||||
std::optional<Error> ex = {});
|
||||
|
||||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
#include "drv-output-substitution-goal.hh"
|
||||
#include "finally.hh"
|
||||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
: Goal(worker)
|
||||
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
|
||||
const DrvOutput & id,
|
||||
Worker & worker,
|
||||
RepairFlag repair,
|
||||
std::optional<ContentAddress> ca)
|
||||
: Goal(worker, DerivedPath::Opaque { StorePath::dummy })
|
||||
, id(id)
|
||||
{
|
||||
state = &DrvOutputSubstitutionGoal::init;
|
||||
|
|
@ -30,7 +36,7 @@ void DrvOutputSubstitutionGoal::init()
|
|||
|
||||
void DrvOutputSubstitutionGoal::tryNext()
|
||||
{
|
||||
trace("Trying next substituter");
|
||||
trace("trying next substituter");
|
||||
|
||||
if (subs.size() == 0) {
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
|
|
@ -50,14 +56,42 @@ void DrvOutputSubstitutionGoal::tryNext()
|
|||
return;
|
||||
}
|
||||
|
||||
auto sub = subs.front();
|
||||
sub = subs.front();
|
||||
subs.pop_front();
|
||||
|
||||
// FIXME: Make async
|
||||
outputInfo = sub->queryRealisation(id);
|
||||
// outputInfo = sub->queryRealisation(id);
|
||||
outPipe.create();
|
||||
promise = decltype(promise)();
|
||||
|
||||
sub->queryRealisation(
|
||||
id, { [&](std::future<std::shared_ptr<const Realisation>> res) {
|
||||
try {
|
||||
Finally updateStats([this]() { outPipe.writeSide.close(); });
|
||||
promise.set_value(res.get());
|
||||
} catch (...) {
|
||||
promise.set_exception(std::current_exception());
|
||||
}
|
||||
} });
|
||||
|
||||
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
|
||||
|
||||
state = &DrvOutputSubstitutionGoal::realisationFetched;
|
||||
}
|
||||
|
||||
void DrvOutputSubstitutionGoal::realisationFetched()
|
||||
{
|
||||
worker.childTerminated(this);
|
||||
|
||||
try {
|
||||
outputInfo = promise.get_future().get();
|
||||
} catch (std::exception & e) {
|
||||
printError(e.what());
|
||||
substituterFailed = true;
|
||||
}
|
||||
|
||||
if (!outputInfo) {
|
||||
tryNext();
|
||||
return;
|
||||
return tryNext();
|
||||
}
|
||||
|
||||
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
|
||||
|
|
@ -89,7 +123,7 @@ void DrvOutputSubstitutionGoal::tryNext()
|
|||
void DrvOutputSubstitutionGoal::outPathValid()
|
||||
{
|
||||
assert(outputInfo);
|
||||
trace("Output path substituted");
|
||||
trace("output path substituted");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
|
|
@ -107,7 +141,7 @@ void DrvOutputSubstitutionGoal::finished()
|
|||
amDone(ecSuccess);
|
||||
}
|
||||
|
||||
string DrvOutputSubstitutionGoal::key()
|
||||
std::string DrvOutputSubstitutionGoal::key()
|
||||
{
|
||||
/* "a$" ensures substitution goals happen before derivation
|
||||
goals. */
|
||||
|
|
@ -119,4 +153,10 @@ void DrvOutputSubstitutionGoal::work()
|
|||
(this->*state)();
|
||||
}
|
||||
|
||||
void DrvOutputSubstitutionGoal::handleEOF(int fd)
|
||||
{
|
||||
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
#include "store-api.hh"
|
||||
#include "goal.hh"
|
||||
#include "realisation.hh"
|
||||
#include <thread>
|
||||
#include <future>
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -20,11 +22,18 @@ private:
|
|||
|
||||
// The realisation corresponding to the given output id.
|
||||
// Will be filled once we can get it.
|
||||
std::optional<Realisation> outputInfo;
|
||||
std::shared_ptr<const Realisation> outputInfo;
|
||||
|
||||
/* The remaining substituters. */
|
||||
std::list<ref<Store>> subs;
|
||||
|
||||
/* The current substituter. */
|
||||
std::shared_ptr<Store> sub;
|
||||
|
||||
Pipe outPipe;
|
||||
std::thread thr;
|
||||
std::promise<std::shared_ptr<const Realisation>> promise;
|
||||
|
||||
/* Whether a substituter failed. */
|
||||
bool substituterFailed = false;
|
||||
|
||||
|
|
@ -36,15 +45,16 @@ public:
|
|||
|
||||
void init();
|
||||
void tryNext();
|
||||
void realisationFetched();
|
||||
void outPathValid();
|
||||
void finished();
|
||||
|
||||
void timedOut(Error && ex) override { abort(); };
|
||||
|
||||
string key() override;
|
||||
std::string key() override;
|
||||
|
||||
void work() override;
|
||||
|
||||
void handleEOF(int fd) override;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
#include "machines.hh"
|
||||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
#include "derivation-goal.hh"
|
||||
|
|
@ -48,43 +47,51 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
|
|||
}
|
||||
}
|
||||
|
||||
std::vector<BuildResult> Store::buildPathsWithResults(
|
||||
const std::vector<DerivedPath> & reqs,
|
||||
BuildMode buildMode,
|
||||
std::shared_ptr<Store> evalStore)
|
||||
{
|
||||
Worker worker(*this, evalStore ? *evalStore : *this);
|
||||
|
||||
Goals goals;
|
||||
for (const auto & br : reqs) {
|
||||
std::visit(overloaded {
|
||||
[&](const DerivedPath::Built & bfd) {
|
||||
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
|
||||
},
|
||||
[&](const DerivedPath::Opaque & bo) {
|
||||
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
|
||||
},
|
||||
}, br.raw());
|
||||
}
|
||||
|
||||
worker.run(goals);
|
||||
|
||||
std::vector<BuildResult> results;
|
||||
|
||||
for (auto & i : goals)
|
||||
results.push_back(i->buildResult);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
BuildMode buildMode)
|
||||
{
|
||||
Worker worker(*this, *this);
|
||||
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
|
||||
|
||||
BuildResult result;
|
||||
|
||||
try {
|
||||
worker.run(Goals{goal});
|
||||
result = goal->getResult();
|
||||
return goal->buildResult;
|
||||
} catch (Error & e) {
|
||||
result.status = BuildResult::MiscFailure;
|
||||
result.errorMsg = e.msg();
|
||||
}
|
||||
// XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
|
||||
// extended to return the full realisation for each output
|
||||
auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
|
||||
auto outputHashes = staticOutputHashes(*this, drv);
|
||||
for (auto & [outputName, staticOutput] : staticDrvOutputs) {
|
||||
auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
|
||||
if (staticOutput.second)
|
||||
result.builtOutputs.insert_or_assign(
|
||||
outputId,
|
||||
Realisation{ outputId, *staticOutput.second}
|
||||
);
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
|
||||
auto realisation = this->queryRealisation(outputId);
|
||||
if (realisation)
|
||||
result.builtOutputs.insert_or_assign(
|
||||
outputId,
|
||||
*realisation
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
return BuildResult {
|
||||
.status = BuildResult::MiscFailure,
|
||||
.errorMsg = e.msg(),
|
||||
.path = DerivedPath::Built { .drvPath = drvPath },
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ namespace nix {
|
|||
|
||||
|
||||
bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
|
||||
string s1 = a->key();
|
||||
string s2 = b->key();
|
||||
std::string s1 = a->key();
|
||||
std::string s2 = b->key();
|
||||
return s1 < s2;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
#include "types.hh"
|
||||
#include "store-api.hh"
|
||||
#include "build-result.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -18,8 +19,8 @@ struct CompareGoalPtrs {
|
|||
};
|
||||
|
||||
/* Set of goals. */
|
||||
typedef set<GoalPtr, CompareGoalPtrs> Goals;
|
||||
typedef set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
|
||||
typedef std::set<GoalPtr, CompareGoalPtrs> Goals;
|
||||
typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
|
||||
|
||||
/* A map of paths to goals (and the other way around). */
|
||||
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
|
||||
|
|
@ -50,15 +51,20 @@ struct Goal : public std::enable_shared_from_this<Goal>
|
|||
unsigned int nrIncompleteClosure;
|
||||
|
||||
/* Name of this goal for debugging purposes. */
|
||||
string name;
|
||||
std::string name;
|
||||
|
||||
/* Whether the goal is finished. */
|
||||
ExitCode exitCode;
|
||||
|
||||
/* Build result. */
|
||||
BuildResult buildResult;
|
||||
|
||||
/* Exception containing an error message, if any. */
|
||||
std::optional<Error> ex;
|
||||
|
||||
Goal(Worker & worker) : worker(worker)
|
||||
Goal(Worker & worker, DerivedPath path)
|
||||
: worker(worker)
|
||||
, buildResult { .path = std::move(path) }
|
||||
{
|
||||
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
|
||||
exitCode = ecBusy;
|
||||
|
|
@ -75,7 +81,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
|
|||
|
||||
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
|
||||
|
||||
virtual void handleChildOutput(int fd, const string & data)
|
||||
virtual void handleChildOutput(int fd, std::string_view data)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
|
@ -87,7 +93,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
|
|||
|
||||
void trace(const FormatOrString & fs);
|
||||
|
||||
string getName()
|
||||
std::string getName()
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
|
@ -97,7 +103,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
|
|||
by the worker (important!), etc. */
|
||||
virtual void timedOut(Error && ex) = 0;
|
||||
|
||||
virtual string key() = 0;
|
||||
virtual std::string key() = 0;
|
||||
|
||||
void amDone(ExitCode result, std::optional<Error> ex = {});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#include "local-derivation-goal.hh"
|
||||
#include "gc-store.hh"
|
||||
#include "hook-instance.hh"
|
||||
#include "worker.hh"
|
||||
#include "builtins.hh"
|
||||
|
|
@ -193,7 +194,7 @@ void LocalDerivationGoal::tryLocalBuild() {
|
|||
outputLocks.unlock();
|
||||
buildUser.reset();
|
||||
worker.permanentFailure = true;
|
||||
done(BuildResult::InputRejected, e);
|
||||
done(BuildResult::InputRejected, {}, e);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -260,6 +261,7 @@ void LocalDerivationGoal::cleanupHookFinally()
|
|||
void LocalDerivationGoal::cleanupPreChildKill()
|
||||
{
|
||||
sandboxMountNamespace = -1;
|
||||
sandboxUserNamespace = -1;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -342,7 +344,7 @@ int childEntry(void * arg)
|
|||
return 1;
|
||||
}
|
||||
|
||||
|
||||
#if __linux__
|
||||
static void linkOrCopy(const Path & from, const Path & to)
|
||||
{
|
||||
if (link(from.c_str(), to.c_str()) == -1) {
|
||||
|
|
@ -358,6 +360,7 @@ static void linkOrCopy(const Path & from, const Path & to)
|
|||
copyPath(from, to);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
void LocalDerivationGoal::startBuilder()
|
||||
|
|
@ -479,12 +482,12 @@ void LocalDerivationGoal::startBuilder()
|
|||
temporary build directory. The text files have the format used
|
||||
by `nix-store --register-validity'. However, the deriver
|
||||
fields are left empty. */
|
||||
string s = get(drv->env, "exportReferencesGraph").value_or("");
|
||||
auto s = get(drv->env, "exportReferencesGraph").value_or("");
|
||||
Strings ss = tokenizeString<Strings>(s);
|
||||
if (ss.size() % 2 != 0)
|
||||
throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
|
||||
for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
|
||||
string fileName = *i++;
|
||||
auto fileName = *i++;
|
||||
static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
|
||||
if (!std::regex_match(fileName, regex))
|
||||
throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
|
||||
|
|
@ -515,10 +518,10 @@ void LocalDerivationGoal::startBuilder()
|
|||
i.pop_back();
|
||||
}
|
||||
size_t p = i.find('=');
|
||||
if (p == string::npos)
|
||||
if (p == std::string::npos)
|
||||
dirsInChroot[i] = {i, optional};
|
||||
else
|
||||
dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
|
||||
dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional};
|
||||
}
|
||||
dirsInChroot[tmpDirInSandbox] = tmpDir;
|
||||
|
||||
|
|
@ -669,9 +672,10 @@ void LocalDerivationGoal::startBuilder()
|
|||
auto state = stBegin;
|
||||
auto lines = runProgram(settings.preBuildHook, false, args);
|
||||
auto lastPos = std::string::size_type{0};
|
||||
for (auto nlPos = lines.find('\n'); nlPos != string::npos;
|
||||
nlPos = lines.find('\n', lastPos)) {
|
||||
auto line = std::string{lines, lastPos, nlPos - lastPos};
|
||||
for (auto nlPos = lines.find('\n'); nlPos != std::string::npos;
|
||||
nlPos = lines.find('\n', lastPos))
|
||||
{
|
||||
auto line = lines.substr(lastPos, nlPos - lastPos);
|
||||
lastPos = nlPos + 1;
|
||||
if (state == stBegin) {
|
||||
if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
|
||||
|
|
@ -684,10 +688,10 @@ void LocalDerivationGoal::startBuilder()
|
|||
state = stBegin;
|
||||
} else {
|
||||
auto p = line.find('=');
|
||||
if (p == string::npos)
|
||||
if (p == std::string::npos)
|
||||
dirsInChroot[line] = line;
|
||||
else
|
||||
dirsInChroot[string(line, 0, p)] = string(line, p + 1);
|
||||
dirsInChroot[line.substr(0, p)] = line.substr(p + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -711,6 +715,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
if (!builderOut.readSide)
|
||||
throw SysError("opening pseudoterminal master");
|
||||
|
||||
// FIXME: not thread-safe, use ptsname_r
|
||||
std::string slaveName(ptsname(builderOut.readSide.get()));
|
||||
|
||||
if (buildUser) {
|
||||
|
|
@ -751,10 +756,9 @@ void LocalDerivationGoal::startBuilder()
|
|||
if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
|
||||
throw SysError("putting pseudoterminal into raw mode");
|
||||
|
||||
result.startTime = time(0);
|
||||
buildResult.startTime = time(0);
|
||||
|
||||
/* Fork a child to build the package. */
|
||||
ProcessOptions options;
|
||||
|
||||
#if __linux__
|
||||
if (useChroot) {
|
||||
|
|
@ -797,8 +801,6 @@ void LocalDerivationGoal::startBuilder()
|
|||
|
||||
userNamespaceSync.create();
|
||||
|
||||
options.allowVfork = false;
|
||||
|
||||
Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
|
||||
static bool userNamespacesEnabled =
|
||||
pathExists(maxUserNamespaces)
|
||||
|
|
@ -856,7 +858,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
writeFull(builderOut.writeSide.get(),
|
||||
fmt("%d %d\n", usingUserNamespace, child));
|
||||
_exit(0);
|
||||
}, options);
|
||||
});
|
||||
|
||||
int res = helper.wait();
|
||||
if (res != 0 && settings.sandboxFallback) {
|
||||
|
|
@ -907,23 +909,30 @@ void LocalDerivationGoal::startBuilder()
|
|||
"nobody:x:65534:65534:Nobody:/:/noshell\n",
|
||||
sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
|
||||
|
||||
/* Save the mount namespace of the child. We have to do this
|
||||
/* Save the mount- and user namespace of the child. We have to do this
|
||||
*before* the child does a chroot. */
|
||||
sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
|
||||
if (sandboxMountNamespace.get() == -1)
|
||||
throw SysError("getting sandbox mount namespace");
|
||||
|
||||
if (usingUserNamespace) {
|
||||
sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY);
|
||||
if (sandboxUserNamespace.get() == -1)
|
||||
throw SysError("getting sandbox user namespace");
|
||||
}
|
||||
|
||||
/* Signal the builder that we've updated its user namespace. */
|
||||
writeFull(userNamespaceSync.writeSide.get(), "1");
|
||||
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
#if __linux__
|
||||
fallback:
|
||||
options.allowVfork = !buildUser && !drv->isBuiltin();
|
||||
#endif
|
||||
pid = startProcess([&]() {
|
||||
runChild();
|
||||
}, options);
|
||||
});
|
||||
}
|
||||
|
||||
/* parent */
|
||||
|
|
@ -934,7 +943,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
/* Check if setting up the build environment failed. */
|
||||
std::vector<std::string> msgs;
|
||||
while (true) {
|
||||
string msg = [&]() {
|
||||
std::string msg = [&]() {
|
||||
try {
|
||||
return readLine(builderOut.readSide.get());
|
||||
} catch (Error & e) {
|
||||
|
|
@ -946,12 +955,12 @@ void LocalDerivationGoal::startBuilder()
|
|||
throw;
|
||||
}
|
||||
}();
|
||||
if (string(msg, 0, 1) == "\2") break;
|
||||
if (string(msg, 0, 1) == "\1") {
|
||||
if (msg.substr(0, 1) == "\2") break;
|
||||
if (msg.substr(0, 1) == "\1") {
|
||||
FdSource source(builderOut.readSide.get());
|
||||
auto ex = readError(source);
|
||||
ex.addTrace({}, "while setting up the build environment");
|
||||
throw;
|
||||
throw ex;
|
||||
}
|
||||
debug("sandbox setup: " + msg);
|
||||
msgs.push_back(std::move(msg));
|
||||
|
|
@ -983,7 +992,7 @@ void LocalDerivationGoal::initTmpDir() {
|
|||
env[i.first] = i.second;
|
||||
} else {
|
||||
auto hash = hashString(htSHA256, i.first);
|
||||
string fn = ".attr-" + hash.to_string(Base32, false);
|
||||
std::string fn = ".attr-" + hash.to_string(Base32, false);
|
||||
Path p = tmpDir + "/" + fn;
|
||||
writeFile(p, rewriteStrings(i.second, inputRewrites));
|
||||
chownToBuilder(p);
|
||||
|
|
@ -1074,7 +1083,7 @@ void LocalDerivationGoal::writeStructuredAttrs()
|
|||
for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
|
||||
/* The placeholder must have a rewrite, so we use it to cover both the
|
||||
cases where we know or don't know the output path ahead of time. */
|
||||
rewritten[i] = rewriteStrings(v, inputRewrites);
|
||||
rewritten[i] = rewriteStrings((std::string) v, inputRewrites);
|
||||
}
|
||||
|
||||
json["outputs"] = rewritten;
|
||||
|
|
@ -1119,7 +1128,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig
|
|||
/* A wrapper around LocalStore that only allows building/querying of
|
||||
paths that are in the input closures of the build or were added via
|
||||
recursive Nix calls. */
|
||||
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
|
||||
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore
|
||||
{
|
||||
ref<LocalStore> next;
|
||||
|
||||
|
|
@ -1180,9 +1189,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
|
||||
{ throw Error("queryPathFromHashPart"); }
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
|
||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
|
||||
StorePath addToStore(
|
||||
std::string_view name,
|
||||
const Path & srcPath,
|
||||
FileIngestionMethod method,
|
||||
HashType hashAlgo,
|
||||
PathFilter & filter,
|
||||
RepairFlag repair,
|
||||
const StorePathSet & references) override
|
||||
{ throw Error("addToStore"); }
|
||||
|
||||
void addToStore(const ValidPathInfo & info, Source & narSource,
|
||||
|
|
@ -1192,18 +1206,26 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
goal.addDependency(info.path);
|
||||
}
|
||||
|
||||
StorePath addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair = NoRepair) override
|
||||
StorePath addTextToStore(
|
||||
std::string_view name,
|
||||
std::string_view s,
|
||||
const StorePathSet & references,
|
||||
RepairFlag repair = NoRepair) override
|
||||
{
|
||||
auto path = next->addTextToStore(name, s, references, repair);
|
||||
goal.addDependency(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
|
||||
StorePath addToStoreFromDump(
|
||||
Source & dump,
|
||||
std::string_view name,
|
||||
FileIngestionMethod method,
|
||||
HashType hashAlgo,
|
||||
RepairFlag repair,
|
||||
const StorePathSet & references) override
|
||||
{
|
||||
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
|
||||
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
|
||||
goal.addDependency(path);
|
||||
return path;
|
||||
}
|
||||
|
|
@ -1227,16 +1249,27 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
// corresponds to an allowed derivation
|
||||
{ throw Error("registerDrvOutput"); }
|
||||
|
||||
std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
|
||||
void queryRealisationUncached(const DrvOutput & id,
|
||||
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
|
||||
// XXX: This should probably be allowed if the realisation corresponds to
|
||||
// an allowed derivation
|
||||
{
|
||||
if (!goal.isAllowed(id))
|
||||
throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
|
||||
return next->queryRealisation(id);
|
||||
callback(nullptr);
|
||||
next->queryRealisation(id, std::move(callback));
|
||||
}
|
||||
|
||||
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
|
||||
{
|
||||
for (auto & result : buildPathsWithResults(paths, buildMode, evalStore))
|
||||
if (!result.success())
|
||||
result.rethrow();
|
||||
}
|
||||
|
||||
std::vector<BuildResult> buildPathsWithResults(
|
||||
const std::vector<DerivedPath> & paths,
|
||||
BuildMode buildMode = bmNormal,
|
||||
std::shared_ptr<Store> evalStore = nullptr) override
|
||||
{
|
||||
assert(!evalStore);
|
||||
|
||||
|
|
@ -1250,26 +1283,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
|
||||
}
|
||||
|
||||
next->buildPaths(paths, buildMode);
|
||||
auto results = next->buildPathsWithResults(paths, buildMode);
|
||||
|
||||
for (auto & path : paths) {
|
||||
auto p = std::get_if<DerivedPath::Built>(&path);
|
||||
if (!p) continue;
|
||||
auto & bfd = *p;
|
||||
auto drv = readDerivation(bfd.drvPath);
|
||||
auto drvHashes = staticOutputHashes(*this, drv);
|
||||
auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
|
||||
for (auto & [outputName, outputPath] : outputs)
|
||||
if (wantOutput(outputName, bfd.outputs)) {
|
||||
newPaths.insert(outputPath);
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
|
||||
auto thisRealisation = next->queryRealisation(
|
||||
DrvOutput{drvHashes.at(outputName), outputName}
|
||||
);
|
||||
assert(thisRealisation);
|
||||
newRealisations.insert(*thisRealisation);
|
||||
}
|
||||
}
|
||||
for (auto & result : results) {
|
||||
for (auto & [outputName, output] : result.builtOutputs) {
|
||||
newPaths.insert(output.outPath);
|
||||
newRealisations.insert(output);
|
||||
}
|
||||
}
|
||||
|
||||
StorePathSet closure;
|
||||
|
|
@ -1278,6 +1298,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
goal.addDependency(path);
|
||||
for (auto & real : Realisation::closure(*next, newRealisations))
|
||||
goal.addedDrvOutputs.insert(real.id);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
|
|
@ -1323,7 +1345,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
|
||||
void LocalDerivationGoal::startDaemon()
|
||||
{
|
||||
settings.requireExperimentalFeature("recursive-nix");
|
||||
settings.requireExperimentalFeature(Xp::RecursiveNix);
|
||||
|
||||
Store::Params params;
|
||||
params["path-info-cache-size"] = "0";
|
||||
|
|
@ -1356,7 +1378,7 @@ void LocalDerivationGoal::startDaemon()
|
|||
AutoCloseFD remote = accept(daemonSocket.get(),
|
||||
(struct sockaddr *) &remoteAddr, &remoteAddrLen);
|
||||
if (!remote) {
|
||||
if (errno == EINTR) continue;
|
||||
if (errno == EINTR || errno == EAGAIN) continue;
|
||||
if (errno == EINVAL) break;
|
||||
throw SysError("accepting connection");
|
||||
}
|
||||
|
|
@ -1435,6 +1457,9 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
|
|||
child process.*/
|
||||
Pid child(startProcess([&]() {
|
||||
|
||||
if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1))
|
||||
throw SysError("entering sandbox user namespace");
|
||||
|
||||
if (setns(sandboxMountNamespace.get(), 0) == -1)
|
||||
throw SysError("entering sandbox mount namespace");
|
||||
|
||||
|
|
@ -1776,11 +1801,14 @@ void LocalDerivationGoal::runChild()
|
|||
i686-linux build on an x86_64-linux machine. */
|
||||
struct utsname utsbuf;
|
||||
uname(&utsbuf);
|
||||
if (drv->platform == "i686-linux" &&
|
||||
(settings.thisSystem == "x86_64-linux" ||
|
||||
(!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
|
||||
if ((drv->platform == "i686-linux"
|
||||
&& (settings.thisSystem == "x86_64-linux"
|
||||
|| (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
|
||||
|| drv->platform == "armv7l-linux"
|
||||
|| drv->platform == "armv6l-linux")
|
||||
{
|
||||
if (personality(PER_LINUX32) == -1)
|
||||
throw SysError("cannot set i686-linux personality");
|
||||
throw SysError("cannot set 32-bit personality");
|
||||
}
|
||||
|
||||
/* Impersonate a Linux 2.6 machine to get some determinism in
|
||||
|
|
@ -1905,7 +1933,7 @@ void LocalDerivationGoal::runChild()
|
|||
"can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
|
||||
i.first, i.second.source);
|
||||
|
||||
string path = i.first;
|
||||
std::string path = i.first;
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st)) {
|
||||
if (i.second.optional && errno == ENOENT)
|
||||
|
|
@ -1957,7 +1985,7 @@ void LocalDerivationGoal::runChild()
|
|||
args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
|
||||
if (allowLocalNetworking) {
|
||||
args.push_back("-D");
|
||||
args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
|
||||
args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1"));
|
||||
}
|
||||
args.push_back(drv->builder);
|
||||
} else {
|
||||
|
|
@ -1976,7 +2004,7 @@ void LocalDerivationGoal::runChild()
|
|||
args.push_back(rewriteStrings(i, inputRewrites));
|
||||
|
||||
/* Indicate that we managed to set up the build environment. */
|
||||
writeFull(STDERR_FILENO, string("\2\n"));
|
||||
writeFull(STDERR_FILENO, std::string("\2\n"));
|
||||
|
||||
/* Execute the program. This should not return. */
|
||||
if (drv->isBuiltin()) {
|
||||
|
|
@ -1994,7 +2022,7 @@ void LocalDerivationGoal::runChild()
|
|||
else if (drv->builder == "builtin:unpack-channel")
|
||||
builtinUnpackChannel(drv2);
|
||||
else
|
||||
throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
|
||||
throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8));
|
||||
_exit(0);
|
||||
} catch (std::exception & e) {
|
||||
writeFull(STDERR_FILENO, e.what() + std::string("\n"));
|
||||
|
|
@ -2040,7 +2068,7 @@ void LocalDerivationGoal::runChild()
|
|||
}
|
||||
|
||||
|
||||
void LocalDerivationGoal::registerOutputs()
|
||||
DrvOutputs LocalDerivationGoal::registerOutputs()
|
||||
{
|
||||
/* When using a build hook, the build hook can register the output
|
||||
as valid (by doing `nix-store --import'). If so we don't have
|
||||
|
|
@ -2049,10 +2077,8 @@ void LocalDerivationGoal::registerOutputs()
|
|||
We can only early return when the outputs are known a priori. For
|
||||
floating content-addressed derivations this isn't the case.
|
||||
*/
|
||||
if (hook) {
|
||||
DerivationGoal::registerOutputs();
|
||||
return;
|
||||
}
|
||||
if (hook)
|
||||
return DerivationGoal::registerOutputs();
|
||||
|
||||
std::map<std::string, ValidPathInfo> infos;
|
||||
|
||||
|
|
@ -2140,8 +2166,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
|
||||
/* Pass blank Sink as we are not ready to hash data at this stage. */
|
||||
NullSink blank;
|
||||
auto references = worker.store.parseStorePathSet(
|
||||
scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
|
||||
auto references = scanForReferences(blank, actualPath, referenceablePaths);
|
||||
|
||||
outputReferencesIfUnregistered.insert_or_assign(
|
||||
outputName,
|
||||
|
|
@ -2176,6 +2201,8 @@ void LocalDerivationGoal::registerOutputs()
|
|||
|
||||
std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
|
||||
|
||||
OutputPathMap finalOutputs;
|
||||
|
||||
for (auto & outputName : sortedOutputNames) {
|
||||
auto output = drv->outputs.at(outputName);
|
||||
auto & scratchPath = scratchOutputs.at(outputName);
|
||||
|
|
@ -2208,14 +2235,14 @@ void LocalDerivationGoal::registerOutputs()
|
|||
auto rewriteOutput = [&]() {
|
||||
/* Apply hash rewriting if necessary. */
|
||||
if (!outputRewrites.empty()) {
|
||||
warn("rewriting hashes in '%1%'; cross fingers", actualPath);
|
||||
debug("rewriting hashes in '%1%'; cross fingers", actualPath);
|
||||
|
||||
/* FIXME: this is in-memory. */
|
||||
StringSink sink;
|
||||
dumpPath(actualPath, sink);
|
||||
deletePath(actualPath);
|
||||
sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
|
||||
StringSource source(*sink.s);
|
||||
sink.s = rewriteStrings(sink.s, outputRewrites);
|
||||
StringSource source(sink.s);
|
||||
restorePath(actualPath, source);
|
||||
}
|
||||
};
|
||||
|
|
@ -2288,7 +2315,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
StringSink sink;
|
||||
dumpPath(actualPath, sink);
|
||||
RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
|
||||
rsink2(*sink.s);
|
||||
rsink2(sink.s);
|
||||
rsink2.flush();
|
||||
});
|
||||
Path tmpPath = actualPath + ".tmp";
|
||||
|
|
@ -2306,6 +2333,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
};
|
||||
|
||||
ValidPathInfo newInfo = std::visit(overloaded {
|
||||
|
||||
[&](const DerivationOutputInputAddressed & output) {
|
||||
/* input-addressed case */
|
||||
auto requiredFinalPath = output.path;
|
||||
|
|
@ -2322,6 +2350,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
static_cast<PathReferences<StorePath> &>(newInfo0) = rewriteRefs();
|
||||
return newInfo0;
|
||||
},
|
||||
|
||||
[&](const DerivationOutputCAFixed & dof) {
|
||||
auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
|
||||
.method = dof.hash.method,
|
||||
|
|
@ -2344,18 +2373,17 @@ void LocalDerivationGoal::registerOutputs()
|
|||
}
|
||||
return newInfo0;
|
||||
},
|
||||
[&](DerivationOutputCAFloating dof) {
|
||||
|
||||
[&](DerivationOutputCAFloating & dof) {
|
||||
return newInfoFromCA(dof);
|
||||
},
|
||||
[&](DerivationOutputDeferred) {
|
||||
|
||||
[&](DerivationOutputDeferred) -> ValidPathInfo {
|
||||
// No derivation should reach that point without having been
|
||||
// rewritten first
|
||||
assert(false);
|
||||
// Ugly, but the compiler insists on having this return a value
|
||||
// of type `ValidPathInfo` despite the `assert(false)`, so
|
||||
// let's provide it
|
||||
return *(ValidPathInfo*)0;
|
||||
},
|
||||
|
||||
}, output.output);
|
||||
|
||||
/* FIXME: set proper permissions in restorePath() so
|
||||
|
|
@ -2446,7 +2474,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
}
|
||||
|
||||
if (curRound == nrRounds) {
|
||||
localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences()
|
||||
localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences()
|
||||
worker.markContentsGood(newInfo.path);
|
||||
}
|
||||
|
||||
|
|
@ -2466,11 +2494,12 @@ void LocalDerivationGoal::registerOutputs()
|
|||
}
|
||||
|
||||
if (buildMode == bmCheck) {
|
||||
// In case of FOD mismatches on `--check` an error must be thrown as this is also
|
||||
// a source for non-determinism.
|
||||
/* In case of fixed-output derivations, if there are
|
||||
mismatches on `--check` an error must be thrown as this is
|
||||
also a source for non-determinism. */
|
||||
if (delayedException)
|
||||
std::rethrow_exception(delayedException);
|
||||
return;
|
||||
return assertPathValidity();
|
||||
}
|
||||
|
||||
/* Apply output checks. */
|
||||
|
|
@ -2482,7 +2511,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
assert(prevInfos.size() == infos.size());
|
||||
for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
|
||||
if (!(*i == *j)) {
|
||||
result.isNonDeterministic = true;
|
||||
buildResult.isNonDeterministic = true;
|
||||
Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
|
||||
bool prevExists = keepPreviousRound && pathExists(prev);
|
||||
hintformat hint = prevExists
|
||||
|
|
@ -2520,7 +2549,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
|
||||
if (curRound < nrRounds) {
|
||||
prevInfos = std::move(infos);
|
||||
return;
|
||||
return {};
|
||||
}
|
||||
|
||||
/* Remove the .check directories if we're done. FIXME: keep them
|
||||
|
|
@ -2555,17 +2584,24 @@ void LocalDerivationGoal::registerOutputs()
|
|||
means it's safe to link the derivation to the output hash. We must do
|
||||
that for floating CA derivations, which otherwise couldn't be cached,
|
||||
but it's fine to do in all cases. */
|
||||
DrvOutputs builtOutputs;
|
||||
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
|
||||
for (auto& [outputName, newInfo] : infos) {
|
||||
auto thisRealisation = Realisation{
|
||||
.id = DrvOutput{initialOutputs.at(outputName).outputHash,
|
||||
outputName},
|
||||
.outPath = newInfo.path};
|
||||
for (auto & [outputName, newInfo] : infos) {
|
||||
auto thisRealisation = Realisation {
|
||||
.id = DrvOutput {
|
||||
initialOutputs.at(outputName).outputHash,
|
||||
outputName
|
||||
},
|
||||
.outPath = newInfo.path
|
||||
};
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
|
||||
signRealisation(thisRealisation);
|
||||
worker.store.registerDrvOutput(thisRealisation);
|
||||
}
|
||||
builtOutputs.emplace(thisRealisation.id, thisRealisation);
|
||||
}
|
||||
|
||||
return builtOutputs;
|
||||
}
|
||||
|
||||
void LocalDerivationGoal::signRealisation(Realisation & realisation)
|
||||
|
|
@ -2574,7 +2610,7 @@ void LocalDerivationGoal::signRealisation(Realisation & realisation)
|
|||
}
|
||||
|
||||
|
||||
void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
|
||||
void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo> & outputs)
|
||||
{
|
||||
std::map<Path, const ValidPathInfo &> outputsByPath;
|
||||
for (auto & output : outputs)
|
||||
|
|
@ -2646,8 +2682,8 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
|
|||
for (auto & i : *value) {
|
||||
if (worker.store.isStorePath(i))
|
||||
spec.insert(worker.store.parseStorePath(i));
|
||||
else if (finalOutputs.count(i))
|
||||
spec.insert(finalOutputs.at(i));
|
||||
else if (outputs.count(i))
|
||||
spec.insert(outputs.at(i).path);
|
||||
else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
|
||||
}
|
||||
|
||||
|
|
@ -2670,7 +2706,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
|
|||
}
|
||||
|
||||
if (!badPaths.empty()) {
|
||||
string badPathsStr;
|
||||
std::string badPathsStr;
|
||||
for (auto & i : badPaths) {
|
||||
badPathsStr += "\n ";
|
||||
badPathsStr += worker.store.printStorePath(i);
|
||||
|
|
|
|||
|
|
@ -27,9 +27,10 @@ struct LocalDerivationGoal : public DerivationGoal
|
|||
/* Pipe for synchronising updates to the builder namespaces. */
|
||||
Pipe userNamespaceSync;
|
||||
|
||||
/* The mount namespace of the builder, used to add additional
|
||||
/* The mount namespace and user namespace of the builder, used to add additional
|
||||
paths to the sandbox as a result of recursive Nix calls. */
|
||||
AutoCloseFD sandboxMountNamespace;
|
||||
AutoCloseFD sandboxUserNamespace;
|
||||
|
||||
/* On Linux, whether we're doing the build in its own user
|
||||
namespace. */
|
||||
|
|
@ -57,11 +58,11 @@ struct LocalDerivationGoal : public DerivationGoal
|
|||
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
|
||||
DirsInChroot dirsInChroot;
|
||||
|
||||
typedef map<string, string> Environment;
|
||||
typedef map<std::string, std::string> Environment;
|
||||
Environment env;
|
||||
|
||||
#if __APPLE__
|
||||
typedef string SandboxProfile;
|
||||
typedef std::string SandboxProfile;
|
||||
SandboxProfile additionalSandboxProfile;
|
||||
#endif
|
||||
|
||||
|
|
@ -168,7 +169,7 @@ struct LocalDerivationGoal : public DerivationGoal
|
|||
|
||||
/* Check that the derivation outputs all exist and register them
|
||||
as valid. */
|
||||
void registerOutputs() override;
|
||||
DrvOutputs registerOutputs() override;
|
||||
|
||||
void signRealisation(Realisation &) override;
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
namespace nix {
|
||||
|
||||
PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
: Goal(worker)
|
||||
: Goal(worker, DerivedPath::Opaque { storePath })
|
||||
, storePath(storePath)
|
||||
, repair(repair)
|
||||
, ca(ca)
|
||||
|
|
@ -24,6 +24,13 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
|
|||
}
|
||||
|
||||
|
||||
void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status)
|
||||
{
|
||||
buildResult.status = status;
|
||||
amDone(result);
|
||||
}
|
||||
|
||||
|
||||
void PathSubstitutionGoal::work()
|
||||
{
|
||||
(this->*state)();
|
||||
|
|
@ -38,7 +45,7 @@ void PathSubstitutionGoal::init()
|
|||
|
||||
/* If the path already exists we're done. */
|
||||
if (!repair && worker.store.isValidPath(storePath)) {
|
||||
amDone(ecSuccess);
|
||||
done(ecSuccess, BuildResult::AlreadyValid);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -65,7 +72,7 @@ void PathSubstitutionGoal::tryNext()
|
|||
/* Hack: don't indicate failure if there were no substituters.
|
||||
In that case the calling derivation should just do a
|
||||
build. */
|
||||
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
|
||||
done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters);
|
||||
|
||||
if (substituterFailed) {
|
||||
worker.failedSubstitutions++;
|
||||
|
|
@ -141,8 +148,8 @@ void PathSubstitutionGoal::tryNext()
|
|||
only after we've downloaded the path. */
|
||||
if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
|
||||
{
|
||||
warn("substituter '%s' does not have a valid signature for path '%s'",
|
||||
sub->getUri(), worker.store.printStorePath(storePath));
|
||||
warn("the substitute for '%s' from '%s' is not signed by any of the keys in 'trusted-public-keys'",
|
||||
worker.store.printStorePath(storePath), sub->getUri());
|
||||
tryNext();
|
||||
return;
|
||||
}
|
||||
|
|
@ -165,7 +172,9 @@ void PathSubstitutionGoal::referencesValid()
|
|||
|
||||
if (nrFailed > 0) {
|
||||
debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
|
||||
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
|
||||
done(
|
||||
nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
|
||||
BuildResult::DependencyFailed);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -269,11 +278,11 @@ void PathSubstitutionGoal::finished()
|
|||
|
||||
worker.updateProgress();
|
||||
|
||||
amDone(ecSuccess);
|
||||
done(ecSuccess, BuildResult::Substituted);
|
||||
}
|
||||
|
||||
|
||||
void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
|
||||
void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data)
|
||||
{
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,13 +53,15 @@ struct PathSubstitutionGoal : public Goal
|
|||
/* Content address for recomputing store path */
|
||||
std::optional<ContentAddress> ca;
|
||||
|
||||
void done(ExitCode result, BuildResult::Status status);
|
||||
|
||||
public:
|
||||
PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
~PathSubstitutionGoal();
|
||||
|
||||
void timedOut(Error && ex) override { abort(); };
|
||||
|
||||
string key() override
|
||||
std::string key() override
|
||||
{
|
||||
/* "a$" ensures substitution goals happen before derivation
|
||||
goals. */
|
||||
|
|
@ -77,7 +79,7 @@ public:
|
|||
void finished();
|
||||
|
||||
/* Callback used by the worker to write to the log. */
|
||||
void handleChildOutput(int fd, const string & data) override;
|
||||
void handleChildOutput(int fd, std::string_view data) override;
|
||||
void handleEOF(int fd) override;
|
||||
|
||||
void cleanup() override;
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ unsigned Worker::getNrLocalBuilds()
|
|||
}
|
||||
|
||||
|
||||
void Worker::childStarted(GoalPtr goal, const set<int> & fds,
|
||||
void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
|
||||
bool inBuildSlot, bool respectTimeouts)
|
||||
{
|
||||
Child child;
|
||||
|
|
@ -239,7 +239,7 @@ void Worker::run(const Goals & _topGoals)
|
|||
}
|
||||
}
|
||||
|
||||
/* Call queryMissing() efficiently query substitutes. */
|
||||
/* Call queryMissing() to efficiently query substitutes. */
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
uint64_t downloadSize, narSize;
|
||||
store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
|
|
@ -281,11 +281,11 @@ void Worker::run(const Goals & _topGoals)
|
|||
if (getMachines().empty())
|
||||
throw Error("unable to start any build; either increase '--max-jobs' "
|
||||
"or enable remote builds."
|
||||
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
|
||||
"\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
|
||||
else
|
||||
throw Error("unable to start any build; remote machines may not have "
|
||||
"all required system features."
|
||||
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
|
||||
"\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
|
||||
|
||||
}
|
||||
assert(!awake.empty());
|
||||
|
|
@ -377,7 +377,7 @@ void Worker::waitForInput()
|
|||
GoalPtr goal = j->goal.lock();
|
||||
assert(goal);
|
||||
|
||||
set<int> fds2(j->fds);
|
||||
std::set<int> fds2(j->fds);
|
||||
std::vector<unsigned char> buffer(4096);
|
||||
for (auto & k : fds2) {
|
||||
if (pollStatus.at(fdToPollStatus.at(k)).revents) {
|
||||
|
|
@ -394,7 +394,7 @@ void Worker::waitForInput()
|
|||
} else {
|
||||
printMsg(lvlVomit, "%1%: read %2% bytes",
|
||||
goal->getName(), rd);
|
||||
string data((char *) buffer.data(), rd);
|
||||
std::string data((char *) buffer.data(), rd);
|
||||
j->lastOutput = after;
|
||||
goal->handleChildOutput(k, data);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ struct Child
|
|||
{
|
||||
WeakGoalPtr goal;
|
||||
Goal * goal2; // ugly hackery
|
||||
set<int> fds;
|
||||
std::set<int> fds;
|
||||
bool respectTimeouts;
|
||||
bool inBuildSlot;
|
||||
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
|
||||
|
|
@ -167,7 +167,7 @@ public:
|
|||
|
||||
/* Registers a running child process. `inBuildSlot' means that
|
||||
the process counts towards the jobs limit. */
|
||||
void childStarted(GoalPtr goal, const set<int> & fds,
|
||||
void childStarted(GoalPtr goal, const std::set<int> & fds,
|
||||
bool inBuildSlot, bool respectTimeouts);
|
||||
|
||||
/* Unregisters a running child process. `wakeSleepers' should be
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue