mirror of
https://github.com/NixOS/nix.git
synced 2025-11-28 21:21:00 +01:00
Merge remote-tracking branch 'origin/master' into auto-uid-allocation
This commit is contained in:
Eelco Dolstra
commit
b95faccf03
401 changed files with 14006 additions and 5711 deletions
|
|
@ -66,7 +66,7 @@ namespace nix {
|
|||
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker)
|
||||
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
|
||||
, useDerivation(true)
|
||||
, drvPath(drvPath)
|
||||
, wantedOutputs(wantedOutputs)
|
||||
|
|
@ -85,7 +85,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
|
|||
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker)
|
||||
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
|
||||
, useDerivation(false)
|
||||
, drvPath(drvPath)
|
||||
, wantedOutputs(wantedOutputs)
|
||||
|
|
@ -135,7 +135,7 @@ void DerivationGoal::killChild()
|
|||
void DerivationGoal::timedOut(Error && ex)
|
||||
{
|
||||
killChild();
|
||||
done(BuildResult::TimedOut, ex);
|
||||
done(BuildResult::TimedOut, {}, ex);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -182,7 +182,7 @@ void DerivationGoal::loadDerivation()
|
|||
trace("loading derivation");
|
||||
|
||||
if (nrFailed != 0) {
|
||||
done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
|
||||
done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -204,10 +204,33 @@ void DerivationGoal::haveDerivation()
|
|||
{
|
||||
trace("have derivation");
|
||||
|
||||
if (drv->type() == DerivationType::CAFloating)
|
||||
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
|
||||
|
||||
if (!drv->type().hasKnownOutputPaths())
|
||||
settings.requireExperimentalFeature(Xp::CaDerivations);
|
||||
|
||||
retrySubstitution = false;
|
||||
if (!drv->type().isPure()) {
|
||||
settings.requireExperimentalFeature(Xp::ImpureDerivations);
|
||||
|
||||
for (auto & [outputName, output] : drv->outputs) {
|
||||
auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
|
||||
assert(!worker.store.isValidPath(randomPath));
|
||||
initialOutputs.insert({
|
||||
outputName,
|
||||
InitialOutput {
|
||||
.wanted = true,
|
||||
.outputHash = impureOutputHash,
|
||||
.known = InitialOutputStatus {
|
||||
.path = randomPath,
|
||||
.status = PathStatus::Absent
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
gaveUpOnSubstitution();
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto & i : drv->outputsAndOptPaths(worker.store))
|
||||
if (i.second.second)
|
||||
|
|
@ -215,34 +238,23 @@ void DerivationGoal::haveDerivation()
|
|||
|
||||
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
|
||||
for (auto & [outputName, outputHash] : outputHashes)
|
||||
initialOutputs.insert({
|
||||
initialOutputs.insert({
|
||||
outputName,
|
||||
InitialOutput{
|
||||
InitialOutput {
|
||||
.wanted = true, // Will be refined later
|
||||
.outputHash = outputHash
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/* Check what outputs paths are not already valid. */
|
||||
checkPathValidity();
|
||||
bool allValid = true;
|
||||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
if (!status.known || !status.known->isValid()) {
|
||||
allValid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
/* If they are all valid, then we're done. */
|
||||
if (allValid && buildMode == bmNormal) {
|
||||
done(BuildResult::AlreadyValid);
|
||||
done(BuildResult::AlreadyValid, std::move(validOutputs));
|
||||
return;
|
||||
}
|
||||
|
||||
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
|
||||
|
||||
|
||||
/* We are first going to try to create the invalid output paths
|
||||
through substitutes. If that doesn't work, we'll build
|
||||
them. */
|
||||
|
|
@ -276,8 +288,10 @@ void DerivationGoal::outputsSubstitutionTried()
|
|||
{
|
||||
trace("all outputs substituted (maybe)");
|
||||
|
||||
assert(drv->type().isPure());
|
||||
|
||||
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
|
||||
done(BuildResult::TransientFailure,
|
||||
done(BuildResult::TransientFailure, {},
|
||||
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
|
||||
worker.store.printStorePath(drvPath)));
|
||||
return;
|
||||
|
|
@ -301,23 +315,17 @@ void DerivationGoal::outputsSubstitutionTried()
|
|||
return;
|
||||
}
|
||||
|
||||
checkPathValidity();
|
||||
size_t nrInvalid = 0;
|
||||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
if (!status.known || !status.known->isValid())
|
||||
nrInvalid++;
|
||||
}
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
if (buildMode == bmNormal && nrInvalid == 0) {
|
||||
done(BuildResult::Substituted);
|
||||
if (buildMode == bmNormal && allValid) {
|
||||
done(BuildResult::Substituted, std::move(validOutputs));
|
||||
return;
|
||||
}
|
||||
if (buildMode == bmRepair && nrInvalid == 0) {
|
||||
if (buildMode == bmRepair && allValid) {
|
||||
repairClosure();
|
||||
return;
|
||||
}
|
||||
if (buildMode == bmCheck && nrInvalid > 0)
|
||||
if (buildMode == bmCheck && !allValid)
|
||||
throw Error("some outputs of '%s' are not valid, so checking is not possible",
|
||||
worker.store.printStorePath(drvPath));
|
||||
|
||||
|
|
@ -325,18 +333,27 @@ void DerivationGoal::outputsSubstitutionTried()
|
|||
gaveUpOnSubstitution();
|
||||
}
|
||||
|
||||
|
||||
/* At least one of the output paths could not be
|
||||
produced using a substitute. So we have to build instead. */
|
||||
void DerivationGoal::gaveUpOnSubstitution()
|
||||
{
|
||||
/* Make sure checkPathValidity() from now on checks all
|
||||
outputs. */
|
||||
wantedOutputs.clear();
|
||||
|
||||
/* The inputs must be built before we can build this goal. */
|
||||
inputDrvOutputs.clear();
|
||||
if (useDerivation)
|
||||
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
|
||||
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
|
||||
/* Ensure that pure, non-fixed-output derivations don't
|
||||
depend on impure derivations. */
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
|
||||
auto inputDrv = worker.evalStore.readDerivation(i.first);
|
||||
if (!inputDrv.type().isPure())
|
||||
throw Error("pure derivation '%s' depends on impure derivation '%s'",
|
||||
worker.store.printStorePath(drvPath),
|
||||
worker.store.printStorePath(i.first));
|
||||
}
|
||||
|
||||
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
|
||||
}
|
||||
|
||||
/* Copy the input sources from the eval store to the build
|
||||
store. */
|
||||
|
|
@ -364,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
|
|||
|
||||
void DerivationGoal::repairClosure()
|
||||
{
|
||||
assert(drv->type().isPure());
|
||||
|
||||
/* If we're repairing, we now know that our own outputs are valid.
|
||||
Now check whether the other paths in the outputs closure are
|
||||
good. If not, then start derivation goals for the derivations
|
||||
|
|
@ -409,7 +428,7 @@ void DerivationGoal::repairClosure()
|
|||
}
|
||||
|
||||
if (waitees.empty()) {
|
||||
done(BuildResult::AlreadyValid);
|
||||
done(BuildResult::AlreadyValid, assertPathValidity());
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -423,7 +442,7 @@ void DerivationGoal::closureRepaired()
|
|||
if (nrFailed > 0)
|
||||
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
|
||||
worker.store.printStorePath(drvPath));
|
||||
done(BuildResult::AlreadyValid);
|
||||
done(BuildResult::AlreadyValid, assertPathValidity());
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -434,13 +453,14 @@ void DerivationGoal::inputsRealised()
|
|||
if (nrFailed != 0) {
|
||||
if (!useDerivation)
|
||||
throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
|
||||
done(BuildResult::DependencyFailed, Error(
|
||||
done(BuildResult::DependencyFailed, {}, Error(
|
||||
"%s dependencies of derivation '%s' failed to build",
|
||||
nrFailed, worker.store.printStorePath(drvPath)));
|
||||
return;
|
||||
}
|
||||
|
||||
if (retrySubstitution) {
|
||||
if (retrySubstitution && !retriedSubstitution) {
|
||||
retriedSubstitution = true;
|
||||
haveDerivation();
|
||||
return;
|
||||
}
|
||||
|
|
@ -454,19 +474,40 @@ void DerivationGoal::inputsRealised()
|
|||
if (useDerivation) {
|
||||
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
|
||||
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
|
||||
((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
|
||||
|| fullDrv.type() == DerivationType::DeferredInputAddressed)) {
|
||||
auto drvType = fullDrv.type();
|
||||
bool resolveDrv = std::visit(overloaded {
|
||||
[&](const DerivationType::InputAddressed & ia) {
|
||||
/* must resolve if deferred. */
|
||||
return ia.deferred;
|
||||
},
|
||||
[&](const DerivationType::ContentAddressed & ca) {
|
||||
return !fullDrv.inputDrvs.empty() && (
|
||||
ca.fixed
|
||||
/* Can optionally resolve if fixed, which is good
|
||||
for avoiding unnecessary rebuilds. */
|
||||
? settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
|
||||
/* Must resolve if floating and there are any inputs
|
||||
drvs. */
|
||||
: true);
|
||||
},
|
||||
[&](const DerivationType::Impure &) {
|
||||
return true;
|
||||
}
|
||||
}, drvType.raw());
|
||||
|
||||
if (resolveDrv && !fullDrv.inputDrvs.empty()) {
|
||||
settings.requireExperimentalFeature(Xp::CaDerivations);
|
||||
|
||||
/* We are be able to resolve this derivation based on the
|
||||
now-known results of dependencies. If so, we become a stub goal
|
||||
aliasing that resolved derivation goal */
|
||||
std::optional attempt = fullDrv.tryResolve(worker.store);
|
||||
now-known results of dependencies. If so, we become a
|
||||
stub goal aliasing that resolved derivation goal. */
|
||||
std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
|
||||
assert(attempt);
|
||||
Derivation drvResolved { *std::move(attempt) };
|
||||
|
||||
auto pathResolved = writeDerivation(worker.store, drvResolved);
|
||||
|
||||
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
|
||||
auto msg = fmt("resolved derivation: '%s' -> '%s'",
|
||||
worker.store.printStorePath(drvPath),
|
||||
worker.store.printStorePath(pathResolved));
|
||||
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
|
||||
|
|
@ -487,21 +528,13 @@ void DerivationGoal::inputsRealised()
|
|||
/* Add the relevant output closures of the input derivation
|
||||
`i' as input paths. Only add the closures of output paths
|
||||
that are specified as inputs. */
|
||||
assert(worker.evalStore.isValidPath(drvPath));
|
||||
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
|
||||
for (auto & j : wantedDepOutputs) {
|
||||
if (outputs.count(j) > 0) {
|
||||
auto optRealizedInput = outputs.at(j);
|
||||
if (!optRealizedInput)
|
||||
throw Error(
|
||||
"derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
|
||||
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
||||
worker.store.computeFSClosure(*optRealizedInput, inputPaths);
|
||||
} else
|
||||
for (auto & j : wantedDepOutputs)
|
||||
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
|
||||
worker.store.computeFSClosure(*outPath, inputPaths);
|
||||
else
|
||||
throw Error(
|
||||
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
|
||||
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -515,7 +548,7 @@ void DerivationGoal::inputsRealised()
|
|||
|
||||
/* Don't repeat fixed-output derivations since they're already
|
||||
verified by their output hash.*/
|
||||
nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
|
||||
nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1;
|
||||
|
||||
/* Okay, try to build. Note that here we don't wait for a build
|
||||
slot to become available, since we don't need one if there is a
|
||||
|
|
@ -523,10 +556,11 @@ void DerivationGoal::inputsRealised()
|
|||
state = &DerivationGoal::tryToBuild;
|
||||
worker.wakeUp(shared_from_this());
|
||||
|
||||
result = BuildResult();
|
||||
buildResult = BuildResult { .path = buildResult.path };
|
||||
}
|
||||
|
||||
void DerivationGoal::started() {
|
||||
void DerivationGoal::started()
|
||||
{
|
||||
auto msg = fmt(
|
||||
buildMode == bmRepair ? "repairing outputs of '%s'" :
|
||||
buildMode == bmCheck ? "checking outputs of '%s'" :
|
||||
|
|
@ -588,19 +622,12 @@ void DerivationGoal::tryToBuild()
|
|||
omitted, but that would be less efficient.) Note that since we
|
||||
now hold the locks on the output paths, no other process can
|
||||
build this derivation, so no further checks are necessary. */
|
||||
checkPathValidity();
|
||||
bool allValid = true;
|
||||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
if (!status.known || !status.known->isValid()) {
|
||||
allValid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
if (buildMode != bmCheck && allValid) {
|
||||
debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
|
||||
outputLocks.setDeletion(true);
|
||||
done(BuildResult::AlreadyValid);
|
||||
done(BuildResult::AlreadyValid, std::move(validOutputs));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -626,7 +653,7 @@ void DerivationGoal::tryToBuild()
|
|||
/* Yes, it has started doing so. Wait until we get
|
||||
EOF from the hook. */
|
||||
actLock.reset();
|
||||
result.startTime = time(0); // inexact
|
||||
buildResult.startTime = time(0); // inexact
|
||||
state = &DerivationGoal::buildDone;
|
||||
started();
|
||||
return;
|
||||
|
|
@ -678,8 +705,7 @@ static void movePath(const Path & src, const Path & dst)
|
|||
if (changePerm)
|
||||
chmod_(src, st.st_mode | S_IWUSR);
|
||||
|
||||
if (rename(src.c_str(), dst.c_str()))
|
||||
throw SysError("renaming '%1%' to '%2%'", src, dst);
|
||||
renameFile(src, dst);
|
||||
|
||||
if (changePerm)
|
||||
chmod_(dst, st.st_mode);
|
||||
|
|
@ -759,8 +785,7 @@ void runPostBuildHook(
|
|||
Store & store,
|
||||
Logger & logger,
|
||||
const StorePath & drvPath,
|
||||
StorePathSet outputPaths
|
||||
)
|
||||
const StorePathSet & outputPaths)
|
||||
{
|
||||
auto hook = settings.postBuildHook;
|
||||
if (hook == "")
|
||||
|
|
@ -830,8 +855,8 @@ void DerivationGoal::buildDone()
|
|||
|
||||
debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
|
||||
|
||||
result.timesBuilt++;
|
||||
result.stopTime = time(0);
|
||||
buildResult.timesBuilt++;
|
||||
buildResult.stopTime = time(0);
|
||||
|
||||
/* So the child is gone now. */
|
||||
worker.childTerminated(this);
|
||||
|
|
@ -876,11 +901,11 @@ void DerivationGoal::buildDone()
|
|||
|
||||
/* Compute the FS closure of the outputs and register them as
|
||||
being valid. */
|
||||
registerOutputs();
|
||||
auto builtOutputs = registerOutputs();
|
||||
|
||||
StorePathSet outputPaths;
|
||||
for (auto & [_, path] : finalOutputs)
|
||||
outputPaths.insert(path);
|
||||
for (auto & [_, output] : builtOutputs)
|
||||
outputPaths.insert(output.outPath);
|
||||
runPostBuildHook(
|
||||
worker.store,
|
||||
*logger,
|
||||
|
|
@ -888,12 +913,6 @@ void DerivationGoal::buildDone()
|
|||
outputPaths
|
||||
);
|
||||
|
||||
if (buildMode == bmCheck) {
|
||||
cleanupPostOutputsRegisteredModeCheck();
|
||||
done(BuildResult::Built);
|
||||
return;
|
||||
}
|
||||
|
||||
cleanupPostOutputsRegisteredModeNonCheck();
|
||||
|
||||
/* Repeat the build if necessary. */
|
||||
|
|
@ -911,6 +930,8 @@ void DerivationGoal::buildDone()
|
|||
outputLocks.setDeletion(true);
|
||||
outputLocks.unlock();
|
||||
|
||||
done(BuildResult::Built, std::move(builtOutputs));
|
||||
|
||||
} catch (BuildError & e) {
|
||||
outputLocks.unlock();
|
||||
|
||||
|
|
@ -926,71 +947,73 @@ void DerivationGoal::buildDone()
|
|||
st =
|
||||
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
|
||||
statusOk(status) ? BuildResult::OutputRejected :
|
||||
derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
|
||||
!derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure :
|
||||
BuildResult::PermanentFailure;
|
||||
}
|
||||
|
||||
done(st, e);
|
||||
done(st, {}, e);
|
||||
return;
|
||||
}
|
||||
|
||||
done(BuildResult::Built);
|
||||
}
|
||||
|
||||
void DerivationGoal::resolvedFinished() {
|
||||
void DerivationGoal::resolvedFinished()
|
||||
{
|
||||
trace("resolved derivation finished");
|
||||
|
||||
assert(resolvedDrvGoal);
|
||||
auto resolvedDrv = *resolvedDrvGoal->drv;
|
||||
auto & resolvedResult = resolvedDrvGoal->buildResult;
|
||||
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
|
||||
DrvOutputs builtOutputs;
|
||||
|
||||
StorePathSet outputPaths;
|
||||
if (resolvedResult.success()) {
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
|
||||
|
||||
// `wantedOutputs` might be empty, which means “all the outputs”
|
||||
auto realWantedOutputs = wantedOutputs;
|
||||
if (realWantedOutputs.empty())
|
||||
realWantedOutputs = resolvedDrv.outputNames();
|
||||
StorePathSet outputPaths;
|
||||
|
||||
for (auto & wantedOutput : realWantedOutputs) {
|
||||
assert(initialOutputs.count(wantedOutput) != 0);
|
||||
assert(resolvedHashes.count(wantedOutput) != 0);
|
||||
auto realisation = worker.store.queryRealisation(
|
||||
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
|
||||
);
|
||||
// We've just built it, but maybe the build failed, in which case the
|
||||
// realisation won't be there
|
||||
if (realisation) {
|
||||
auto newRealisation = *realisation;
|
||||
newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
|
||||
newRealisation.signatures.clear();
|
||||
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
|
||||
signRealisation(newRealisation);
|
||||
worker.store.registerDrvOutput(newRealisation);
|
||||
// `wantedOutputs` might be empty, which means “all the outputs”
|
||||
auto realWantedOutputs = wantedOutputs;
|
||||
if (realWantedOutputs.empty())
|
||||
realWantedOutputs = resolvedDrv.outputNames();
|
||||
|
||||
for (auto & wantedOutput : realWantedOutputs) {
|
||||
auto initialOutput = get(initialOutputs, wantedOutput);
|
||||
auto resolvedHash = get(resolvedHashes, wantedOutput);
|
||||
if ((!initialOutput) || (!resolvedHash))
|
||||
throw Error(
|
||||
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
|
||||
worker.store.printStorePath(drvPath), wantedOutput);
|
||||
auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput });
|
||||
if (!realisation)
|
||||
throw Error(
|
||||
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
|
||||
worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
|
||||
if (drv->type().isPure()) {
|
||||
auto newRealisation = *realisation;
|
||||
newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
|
||||
newRealisation.signatures.clear();
|
||||
if (!drv->type().isFixed())
|
||||
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
|
||||
signRealisation(newRealisation);
|
||||
worker.store.registerDrvOutput(newRealisation);
|
||||
}
|
||||
outputPaths.insert(realisation->outPath);
|
||||
} else {
|
||||
// If we don't have a realisation, then it must mean that something
|
||||
// failed when building the resolved drv
|
||||
assert(!result.success());
|
||||
builtOutputs.emplace(realisation->id, *realisation);
|
||||
}
|
||||
|
||||
runPostBuildHook(
|
||||
worker.store,
|
||||
*logger,
|
||||
drvPath,
|
||||
outputPaths
|
||||
);
|
||||
}
|
||||
|
||||
runPostBuildHook(
|
||||
worker.store,
|
||||
*logger,
|
||||
drvPath,
|
||||
outputPaths
|
||||
);
|
||||
auto status = resolvedResult.status;
|
||||
if (status == BuildResult::AlreadyValid)
|
||||
status = BuildResult::ResolvesToAlreadyValid;
|
||||
|
||||
auto status = [&]() {
|
||||
auto resolvedResult = resolvedDrvGoal->getResult();
|
||||
switch (resolvedResult.status) {
|
||||
case BuildResult::AlreadyValid:
|
||||
return BuildResult::ResolvesToAlreadyValid;
|
||||
default:
|
||||
return resolvedResult.status;
|
||||
}
|
||||
}();
|
||||
|
||||
done(status);
|
||||
done(status, std::move(builtOutputs));
|
||||
}
|
||||
|
||||
HookReply DerivationGoal::tryBuildHook()
|
||||
|
|
@ -1100,7 +1123,7 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
}
|
||||
|
||||
|
||||
void DerivationGoal::registerOutputs()
|
||||
DrvOutputs DerivationGoal::registerOutputs()
|
||||
{
|
||||
/* When using a build hook, the build hook can register the output
|
||||
as valid (by doing `nix-store --import'). If so we don't have
|
||||
|
|
@ -1109,21 +1132,7 @@ void DerivationGoal::registerOutputs()
|
|||
We can only early return when the outputs are known a priori. For
|
||||
floating content-addressed derivations this isn't the case.
|
||||
*/
|
||||
for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
|
||||
if (!wantOutput(outputName, wantedOutputs))
|
||||
continue;
|
||||
if (!optOutputPath)
|
||||
throw BuildError(
|
||||
"output '%s' from derivation '%s' does not have a known output path",
|
||||
outputName, worker.store.printStorePath(drvPath));
|
||||
auto & outputPath = *optOutputPath;
|
||||
if (!worker.store.isValidPath(outputPath))
|
||||
throw BuildError(
|
||||
"output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid",
|
||||
outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath));
|
||||
|
||||
finalOutputs.insert_or_assign(outputName, outputPath);
|
||||
}
|
||||
return assertPathValidity();
|
||||
}
|
||||
|
||||
Path DerivationGoal::openLogFile()
|
||||
|
|
@ -1175,16 +1184,17 @@ bool DerivationGoal::isReadDesc(int fd)
|
|||
return fd == hook->builderOut.readSide.get();
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::handleChildOutput(int fd, std::string_view data)
|
||||
{
|
||||
if (isReadDesc(fd))
|
||||
// local & `ssh://`-builds are dealt with here.
|
||||
auto isWrittenToLog = isReadDesc(fd);
|
||||
if (isWrittenToLog)
|
||||
{
|
||||
logSize += data.size();
|
||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||
killChild();
|
||||
done(
|
||||
BuildResult::LogLimitExceeded,
|
||||
BuildResult::LogLimitExceeded, {},
|
||||
Error("%s killed after writing more than %d bytes of log output",
|
||||
getName(), settings.maxLogSize));
|
||||
return;
|
||||
|
|
@ -1207,7 +1217,16 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
|
|||
if (hook && fd == hook->fromHook.readSide.get()) {
|
||||
for (auto c : data)
|
||||
if (c == '\n') {
|
||||
handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
|
||||
auto json = parseJSONMessage(currentHookLine);
|
||||
if (json) {
|
||||
auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
|
||||
// ensure that logs from a builder using `ssh-ng://` as protocol
|
||||
// are also available to `nix log`.
|
||||
if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) {
|
||||
auto f = (*json)["fields"];
|
||||
(*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n");
|
||||
}
|
||||
}
|
||||
currentHookLine.clear();
|
||||
} else
|
||||
currentHookLine += c;
|
||||
|
|
@ -1241,7 +1260,8 @@ void DerivationGoal::flushLine()
|
|||
|
||||
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
|
||||
{
|
||||
if (!useDerivation || drv->type() != DerivationType::CAFloating) {
|
||||
assert(drv->type().isPure());
|
||||
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
|
||||
std::map<std::string, std::optional<StorePath>> res;
|
||||
for (auto & [name, output] : drv->outputs)
|
||||
res.insert_or_assign(name, output.path(worker.store, drv->name, name));
|
||||
|
|
@ -1253,7 +1273,8 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
|
|||
|
||||
OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
||||
{
|
||||
if (!useDerivation || drv->type() != DerivationType::CAFloating) {
|
||||
assert(drv->type().isPure());
|
||||
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
|
||||
OutputPathMap res;
|
||||
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
|
||||
res.insert_or_assign(name, *output.second);
|
||||
|
|
@ -1264,12 +1285,20 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
|||
}
|
||||
|
||||
|
||||
void DerivationGoal::checkPathValidity()
|
||||
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
|
||||
{
|
||||
if (!drv->type().isPure()) return { false, {} };
|
||||
|
||||
bool checkHash = buildMode == bmRepair;
|
||||
auto wantedOutputsLeft = wantedOutputs;
|
||||
DrvOutputs validOutputs;
|
||||
|
||||
for (auto & i : queryPartialDerivationOutputMap()) {
|
||||
InitialOutput & info = initialOutputs.at(i.first);
|
||||
auto initialOutput = get(initialOutputs, i.first);
|
||||
if (!initialOutput)
|
||||
// this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
|
||||
continue;
|
||||
auto & info = *initialOutput;
|
||||
info.wanted = wantOutput(i.first, wantedOutputs);
|
||||
if (info.wanted)
|
||||
wantedOutputsLeft.erase(i.first);
|
||||
|
|
@ -1284,27 +1313,30 @@ void DerivationGoal::checkPathValidity()
|
|||
: PathStatus::Corrupt,
|
||||
};
|
||||
}
|
||||
auto drvOutput = DrvOutput{info.outputHash, i.first};
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
|
||||
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
|
||||
if (auto real = worker.store.queryRealisation(drvOutput)) {
|
||||
info.known = {
|
||||
.path = real->outPath,
|
||||
.status = PathStatus::Valid,
|
||||
};
|
||||
} else if (info.known && info.known->status == PathStatus::Valid) {
|
||||
// We know the output because it' a static output of the
|
||||
} else if (info.known && info.known->isValid()) {
|
||||
// We know the output because it's a static output of the
|
||||
// derivation, and the output path is valid, but we don't have
|
||||
// its realisation stored (probably because it has been built
|
||||
// without the `ca-derivations` experimental flag)
|
||||
// without the `ca-derivations` experimental flag).
|
||||
worker.store.registerDrvOutput(
|
||||
Realisation{
|
||||
Realisation {
|
||||
drvOutput,
|
||||
info.known->path,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
if (info.wanted && info.known && info.known->isValid())
|
||||
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
|
||||
}
|
||||
|
||||
// If we requested all the outputs via the empty set, we are always fine.
|
||||
// If we requested specific elements, the loop above removes all the valid
|
||||
// ones, so any that are left must be invalid.
|
||||
|
|
@ -1312,24 +1344,48 @@ void DerivationGoal::checkPathValidity()
|
|||
throw Error("derivation '%s' does not have wanted outputs %s",
|
||||
worker.store.printStorePath(drvPath),
|
||||
concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
|
||||
|
||||
bool allValid = true;
|
||||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.wanted) continue;
|
||||
if (!status.known || !status.known->isValid()) {
|
||||
allValid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { allValid, validOutputs };
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
|
||||
DrvOutputs DerivationGoal::assertPathValidity()
|
||||
{
|
||||
result.status = status;
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
if (!allValid)
|
||||
throw Error("some outputs are unexpectedly invalid");
|
||||
return validOutputs;
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::done(
|
||||
BuildResult::Status status,
|
||||
DrvOutputs builtOutputs,
|
||||
std::optional<Error> ex)
|
||||
{
|
||||
buildResult.status = status;
|
||||
if (ex)
|
||||
result.errorMsg = ex->what();
|
||||
amDone(result.success() ? ecSuccess : ecFailed, ex);
|
||||
if (result.status == BuildResult::TimedOut)
|
||||
buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg));
|
||||
if (buildResult.status == BuildResult::TimedOut)
|
||||
worker.timedOut = true;
|
||||
if (result.status == BuildResult::PermanentFailure)
|
||||
if (buildResult.status == BuildResult::PermanentFailure)
|
||||
worker.permanentFailure = true;
|
||||
|
||||
mcExpectedBuilds.reset();
|
||||
mcRunningBuilds.reset();
|
||||
|
||||
if (result.success()) {
|
||||
if (buildResult.success()) {
|
||||
assert(!builtOutputs.empty());
|
||||
buildResult.builtOutputs = std::move(builtOutputs);
|
||||
if (status == BuildResult::Built)
|
||||
worker.doneBuilds++;
|
||||
} else {
|
||||
|
|
@ -1343,9 +1399,23 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
|
|||
if (traceBuiltOutputsFile != "") {
|
||||
std::fstream fs;
|
||||
fs.open(traceBuiltOutputsFile, std::fstream::out);
|
||||
fs << worker.store.printStorePath(drvPath) << "\t" << result.toString() << std::endl;
|
||||
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
|
||||
}
|
||||
|
||||
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
|
||||
{
|
||||
Goal::waiteeDone(waitee, result);
|
||||
|
||||
if (waitee->buildResult.success())
|
||||
if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
|
||||
for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
|
||||
inputDrvOutputs.insert_or_assign(
|
||||
{ bfd->drvPath, output.outputName },
|
||||
realisation.outPath);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,12 +57,21 @@ struct DerivationGoal : public Goal
|
|||
them. */
|
||||
StringSet wantedOutputs;
|
||||
|
||||
/* Mapping from input derivations + output names to actual store
|
||||
paths. This is filled in by waiteeDone() as each dependency
|
||||
finishes, before inputsRealised() is reached, */
|
||||
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
|
||||
|
||||
/* Whether additional wanted outputs have been added. */
|
||||
bool needRestart = false;
|
||||
|
||||
/* Whether to retry substituting the outputs after building the
|
||||
inputs. */
|
||||
bool retrySubstitution;
|
||||
inputs. This is done in case of an incomplete closure. */
|
||||
bool retrySubstitution = false;
|
||||
|
||||
/* Whether we've retried substitution, in which case we won't try
|
||||
again. */
|
||||
bool retriedSubstitution = false;
|
||||
|
||||
/* The derivation stored at drvPath. */
|
||||
std::unique_ptr<Derivation> drv;
|
||||
|
|
@ -104,20 +113,8 @@ struct DerivationGoal : public Goal
|
|||
typedef void (DerivationGoal::*GoalState)();
|
||||
GoalState state;
|
||||
|
||||
/* The final output paths of the build.
|
||||
|
||||
- For input-addressed derivations, always the precomputed paths
|
||||
|
||||
- For content-addressed derivations, calcuated from whatever the hash
|
||||
ends up being. (Note that fixed outputs derivations that produce the
|
||||
"wrong" output still install that data under its true content-address.)
|
||||
*/
|
||||
OutputPathMap finalOutputs;
|
||||
|
||||
BuildMode buildMode;
|
||||
|
||||
BuildResult result;
|
||||
|
||||
/* The current round, if we're building multiple times. */
|
||||
size_t curRound = 1;
|
||||
|
||||
|
|
@ -152,8 +149,6 @@ struct DerivationGoal : public Goal
|
|||
/* Add wanted outputs to an already existing derivation goal. */
|
||||
void addWantedOutputs(const StringSet & outputs);
|
||||
|
||||
BuildResult getResult() { return result; }
|
||||
|
||||
/* The states. */
|
||||
void getDerivation();
|
||||
void loadDerivation();
|
||||
|
|
@ -175,7 +170,7 @@ struct DerivationGoal : public Goal
|
|||
|
||||
/* Check that the derivation outputs all exist and register them
|
||||
as valid. */
|
||||
virtual void registerOutputs();
|
||||
virtual DrvOutputs registerOutputs();
|
||||
|
||||
/* Open a log file and a pipe to it. */
|
||||
Path openLogFile();
|
||||
|
|
@ -210,8 +205,17 @@ struct DerivationGoal : public Goal
|
|||
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
|
||||
OutputPathMap queryDerivationOutputMap();
|
||||
|
||||
/* Return the set of (in)valid paths. */
|
||||
void checkPathValidity();
|
||||
/* Update 'initialOutputs' to determine the current status of the
|
||||
outputs of the derivation. Also returns a Boolean denoting
|
||||
whether all outputs are valid and non-corrupt, and a
|
||||
'DrvOutputs' structure containing the valid and wanted
|
||||
outputs. */
|
||||
std::pair<bool, DrvOutputs> checkPathValidity();
|
||||
|
||||
/* Aborts if any output is not valid or corrupt, and otherwise
|
||||
returns a 'DrvOutputs' structure containing the wanted
|
||||
outputs. */
|
||||
DrvOutputs assertPathValidity();
|
||||
|
||||
/* Forcibly kill the child process, if any. */
|
||||
virtual void killChild();
|
||||
|
|
@ -222,8 +226,11 @@ struct DerivationGoal : public Goal
|
|||
|
||||
void done(
|
||||
BuildResult::Status status,
|
||||
DrvOutputs builtOutputs = {},
|
||||
std::optional<Error> ex = {});
|
||||
|
||||
void waiteeDone(GoalPtr waitee, ExitCode result) override;
|
||||
|
||||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -6,8 +6,12 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
: Goal(worker)
|
||||
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
|
||||
const DrvOutput & id,
|
||||
Worker & worker,
|
||||
RepairFlag repair,
|
||||
std::optional<ContentAddress> ca)
|
||||
: Goal(worker, DerivedPath::Opaque { StorePath::dummy })
|
||||
, id(id)
|
||||
{
|
||||
state = &DrvOutputSubstitutionGoal::init;
|
||||
|
|
@ -32,12 +36,12 @@ void DrvOutputSubstitutionGoal::init()
|
|||
|
||||
void DrvOutputSubstitutionGoal::tryNext()
|
||||
{
|
||||
trace("Trying next substituter");
|
||||
trace("trying next substituter");
|
||||
|
||||
if (subs.size() == 0) {
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
with it. */
|
||||
debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
|
||||
debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
|
||||
|
||||
/* Hack: don't indicate failure if there were no substituters.
|
||||
In that case the calling derivation should just do a
|
||||
|
|
@ -119,7 +123,7 @@ void DrvOutputSubstitutionGoal::realisationFetched()
|
|||
void DrvOutputSubstitutionGoal::outPathValid()
|
||||
{
|
||||
assert(outputInfo);
|
||||
trace("Output path substituted");
|
||||
trace("output path substituted");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
|
|
|
|||
|
|
@ -47,43 +47,51 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
|
|||
}
|
||||
}
|
||||
|
||||
std::vector<BuildResult> Store::buildPathsWithResults(
|
||||
const std::vector<DerivedPath> & reqs,
|
||||
BuildMode buildMode,
|
||||
std::shared_ptr<Store> evalStore)
|
||||
{
|
||||
Worker worker(*this, evalStore ? *evalStore : *this);
|
||||
|
||||
Goals goals;
|
||||
for (const auto & br : reqs) {
|
||||
std::visit(overloaded {
|
||||
[&](const DerivedPath::Built & bfd) {
|
||||
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
|
||||
},
|
||||
[&](const DerivedPath::Opaque & bo) {
|
||||
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
|
||||
},
|
||||
}, br.raw());
|
||||
}
|
||||
|
||||
worker.run(goals);
|
||||
|
||||
std::vector<BuildResult> results;
|
||||
|
||||
for (auto & i : goals)
|
||||
results.push_back(i->buildResult);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
BuildMode buildMode)
|
||||
{
|
||||
Worker worker(*this, *this);
|
||||
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
|
||||
|
||||
BuildResult result;
|
||||
|
||||
try {
|
||||
worker.run(Goals{goal});
|
||||
result = goal->getResult();
|
||||
return goal->buildResult;
|
||||
} catch (Error & e) {
|
||||
result.status = BuildResult::MiscFailure;
|
||||
result.errorMsg = e.msg();
|
||||
}
|
||||
// XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
|
||||
// extended to return the full realisation for each output
|
||||
auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
|
||||
auto outputHashes = staticOutputHashes(*this, drv);
|
||||
for (auto & [outputName, staticOutput] : staticDrvOutputs) {
|
||||
auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
|
||||
if (staticOutput.second)
|
||||
result.builtOutputs.insert_or_assign(
|
||||
outputId,
|
||||
Realisation{ outputId, *staticOutput.second}
|
||||
);
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
|
||||
auto realisation = this->queryRealisation(outputId);
|
||||
if (realisation)
|
||||
result.builtOutputs.insert_or_assign(
|
||||
outputId,
|
||||
*realisation
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
return BuildResult {
|
||||
.status = BuildResult::MiscFailure,
|
||||
.errorMsg = e.msg(),
|
||||
.path = DerivedPath::Built { .drvPath = drvPath },
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ void Goal::addWaitee(GoalPtr waitee)
|
|||
|
||||
void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
|
||||
{
|
||||
assert(waitees.find(waitee) != waitees.end());
|
||||
assert(waitees.count(waitee));
|
||||
waitees.erase(waitee);
|
||||
|
||||
trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
#include "types.hh"
|
||||
#include "store-api.hh"
|
||||
#include "build-result.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -39,30 +40,32 @@ struct Goal : public std::enable_shared_from_this<Goal>
|
|||
WeakGoals waiters;
|
||||
|
||||
/* Number of goals we are/were waiting for that have failed. */
|
||||
unsigned int nrFailed;
|
||||
size_t nrFailed = 0;
|
||||
|
||||
/* Number of substitution goals we are/were waiting for that
|
||||
failed because there are no substituters. */
|
||||
unsigned int nrNoSubstituters;
|
||||
size_t nrNoSubstituters = 0;
|
||||
|
||||
/* Number of substitution goals we are/were waiting for that
|
||||
failed because they had unsubstitutable references. */
|
||||
unsigned int nrIncompleteClosure;
|
||||
size_t nrIncompleteClosure = 0;
|
||||
|
||||
/* Name of this goal for debugging purposes. */
|
||||
std::string name;
|
||||
|
||||
/* Whether the goal is finished. */
|
||||
ExitCode exitCode;
|
||||
ExitCode exitCode = ecBusy;
|
||||
|
||||
/* Build result. */
|
||||
BuildResult buildResult;
|
||||
|
||||
/* Exception containing an error message, if any. */
|
||||
std::optional<Error> ex;
|
||||
|
||||
Goal(Worker & worker) : worker(worker)
|
||||
{
|
||||
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
|
||||
exitCode = ecBusy;
|
||||
}
|
||||
Goal(Worker & worker, DerivedPath path)
|
||||
: worker(worker)
|
||||
, buildResult { .path = std::move(path) }
|
||||
{ }
|
||||
|
||||
virtual ~Goal()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -7,6 +7,22 @@ HookInstance::HookInstance()
|
|||
{
|
||||
debug("starting build hook '%s'", settings.buildHook);
|
||||
|
||||
auto buildHookArgs = tokenizeString<std::list<std::string>>(settings.buildHook.get());
|
||||
|
||||
if (buildHookArgs.empty())
|
||||
throw Error("'build-hook' setting is empty");
|
||||
|
||||
auto buildHook = buildHookArgs.front();
|
||||
buildHookArgs.pop_front();
|
||||
|
||||
Strings args;
|
||||
args.push_back(std::string(baseNameOf(buildHook)));
|
||||
|
||||
for (auto & arg : buildHookArgs)
|
||||
args.push_back(arg);
|
||||
|
||||
args.push_back(std::to_string(verbosity));
|
||||
|
||||
/* Create a pipe to get the output of the child. */
|
||||
fromHook.create();
|
||||
|
||||
|
|
@ -36,14 +52,9 @@ HookInstance::HookInstance()
|
|||
if (dup2(builderOut.readSide.get(), 5) == -1)
|
||||
throw SysError("dupping builder's stdout/stderr");
|
||||
|
||||
Strings args = {
|
||||
std::string(baseNameOf(settings.buildHook.get())),
|
||||
std::to_string(verbosity),
|
||||
};
|
||||
execv(buildHook.c_str(), stringsToCharPtrs(args).data());
|
||||
|
||||
execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
|
||||
|
||||
throw SysError("executing '%s'", settings.buildHook);
|
||||
throw SysError("executing '%s'", buildHook);
|
||||
});
|
||||
|
||||
pid.setSeparatePG(true);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#include "local-derivation-goal.hh"
|
||||
#include "gc-store.hh"
|
||||
#include "hook-instance.hh"
|
||||
#include "worker.hh"
|
||||
#include "builtins.hh"
|
||||
|
|
@ -13,6 +14,7 @@
|
|||
#include "worker-protocol.hh"
|
||||
#include "topo-sort.hh"
|
||||
#include "callback.hh"
|
||||
#include "json-utils.hh"
|
||||
|
||||
#include <regex>
|
||||
#include <queue>
|
||||
|
|
@ -55,8 +57,6 @@
|
|||
#include <pwd.h>
|
||||
#include <grp.h>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix {
|
||||
|
||||
void handleDiffHook(
|
||||
|
|
@ -186,7 +186,7 @@ void LocalDerivationGoal::tryLocalBuild() {
|
|||
outputLocks.unlock();
|
||||
buildUser.reset();
|
||||
worker.permanentFailure = true;
|
||||
done(BuildResult::InputRejected, e);
|
||||
done(BuildResult::InputRejected, {}, e);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -216,8 +216,7 @@ static void movePath(const Path & src, const Path & dst)
|
|||
if (changePerm)
|
||||
chmod_(src, st.st_mode | S_IWUSR);
|
||||
|
||||
if (rename(src.c_str(), dst.c_str()))
|
||||
throw SysError("renaming '%1%' to '%2%'", src, dst);
|
||||
renameFile(src, dst);
|
||||
|
||||
if (changePerm)
|
||||
chmod_(dst, st.st_mode);
|
||||
|
|
@ -304,7 +303,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
|
|||
if (buildMode != bmCheck && status.known->isValid()) continue;
|
||||
auto p = worker.store.printStorePath(status.known->path);
|
||||
if (pathExists(chrootRootDir + p))
|
||||
rename((chrootRootDir + p).c_str(), p.c_str());
|
||||
renameFile((chrootRootDir + p), p);
|
||||
}
|
||||
|
||||
return diskFull;
|
||||
|
|
@ -387,7 +386,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
else if (settings.sandboxMode == smDisabled)
|
||||
useChroot = false;
|
||||
else if (settings.sandboxMode == smRelaxed)
|
||||
useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
|
||||
useChroot = derivationType.isSandboxed() && !noChroot;
|
||||
}
|
||||
|
||||
auto & localStore = getLocalStore();
|
||||
|
|
@ -474,7 +473,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
temporary build directory. The text files have the format used
|
||||
by `nix-store --register-validity'. However, the deriver
|
||||
fields are left empty. */
|
||||
auto s = get(drv->env, "exportReferencesGraph").value_or("");
|
||||
auto s = getOr(drv->env, "exportReferencesGraph", "");
|
||||
Strings ss = tokenizeString<Strings>(s);
|
||||
if (ss.size() % 2 != 0)
|
||||
throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
|
||||
|
|
@ -608,7 +607,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
"nogroup:x:65534:\n", sandboxGid()));
|
||||
|
||||
/* Create /etc/hosts with localhost entry. */
|
||||
if (!(derivationIsImpure(derivationType)))
|
||||
if (derivationType.isSandboxed())
|
||||
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
|
||||
|
||||
/* Make the closure of the inputs available in the chroot,
|
||||
|
|
@ -724,6 +723,9 @@ void LocalDerivationGoal::startBuilder()
|
|||
|
||||
/* Run the builder. */
|
||||
printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
|
||||
printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args));
|
||||
for (auto & i : drv->env)
|
||||
printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second);
|
||||
|
||||
/* Create the log file. */
|
||||
Path logFile = openLogFile();
|
||||
|
|
@ -776,7 +778,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
|
||||
throw SysError("putting pseudoterminal into raw mode");
|
||||
|
||||
result.startTime = time(0);
|
||||
buildResult.startTime = time(0);
|
||||
|
||||
/* Fork a child to build the package. */
|
||||
|
||||
|
|
@ -816,7 +818,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
us.
|
||||
*/
|
||||
|
||||
if (!(derivationIsImpure(derivationType)))
|
||||
if (derivationType.isSandboxed())
|
||||
privateNetwork = true;
|
||||
|
||||
userNamespaceSync.create();
|
||||
|
|
@ -863,18 +865,43 @@ void LocalDerivationGoal::startBuilder()
|
|||
/* Some distros patch Linux to not allow unprivileged
|
||||
* user namespaces. If we get EPERM or EINVAL, try
|
||||
* without CLONE_NEWUSER and see if that works.
|
||||
* Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6
|
||||
*/
|
||||
usingUserNamespace = false;
|
||||
flags &= ~CLONE_NEWUSER;
|
||||
child = clone(childEntry, stack + stackSize, flags, this);
|
||||
}
|
||||
/* Otherwise exit with EPERM so we can handle this in the
|
||||
parent. This is only done when sandbox-fallback is set
|
||||
to true (the default). */
|
||||
if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
|
||||
_exit(1);
|
||||
if (child == -1) throw SysError("cloning builder process");
|
||||
|
||||
if (child == -1) {
|
||||
switch(errno) {
|
||||
case EPERM:
|
||||
case EINVAL: {
|
||||
int errno_ = errno;
|
||||
if (!userNamespacesEnabled && errno==EPERM)
|
||||
notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces");
|
||||
if (userNamespacesEnabled) {
|
||||
Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone";
|
||||
if (pathExists(procSysKernelUnprivilegedUsernsClone)
|
||||
&& trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") {
|
||||
notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone");
|
||||
}
|
||||
}
|
||||
Path procSelfNsUser = "/proc/self/ns/user";
|
||||
if (!pathExists(procSelfNsUser))
|
||||
notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing");
|
||||
/* Otherwise exit with EPERM so we can handle this in the
|
||||
parent. This is only done when sandbox-fallback is set
|
||||
to true (the default). */
|
||||
if (settings.sandboxFallback)
|
||||
_exit(1);
|
||||
/* Mention sandbox-fallback in the error message so the user
|
||||
knows that having it disabled contributed to the
|
||||
unrecoverability of this failure */
|
||||
throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback");
|
||||
}
|
||||
default:
|
||||
throw SysError("creating sandboxed builder process using clone()");
|
||||
}
|
||||
}
|
||||
writeFull(builderOut.writeSide.get(),
|
||||
fmt("%d %d\n", usingUserNamespace, child));
|
||||
_exit(0);
|
||||
|
|
@ -1014,7 +1041,7 @@ void LocalDerivationGoal::initTmpDir() {
|
|||
there is no size constraint). */
|
||||
if (!parsedDrv->getStructuredAttrs()) {
|
||||
|
||||
StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
|
||||
StringSet passAsFile = tokenizeString<StringSet>(getOr(drv->env, "passAsFile", ""));
|
||||
for (auto & i : drv->env) {
|
||||
if (passAsFile.find(i.first) == passAsFile.end()) {
|
||||
env[i.first] = i.second;
|
||||
|
|
@ -1077,7 +1104,7 @@ void LocalDerivationGoal::initEnv()
|
|||
derivation, tell the builder, so that for instance `fetchurl'
|
||||
can skip checking the output. On older Nixes, this environment
|
||||
variable won't be set, so `fetchurl' will do the check. */
|
||||
if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
|
||||
if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1";
|
||||
|
||||
/* *Only* if this is a fixed-output derivation, propagate the
|
||||
values of the environment variables specified in the
|
||||
|
|
@ -1088,7 +1115,7 @@ void LocalDerivationGoal::initEnv()
|
|||
to the builder is generally impure, but the output of
|
||||
fixed-output derivations is by definition pure (since we
|
||||
already know the cryptographic hash of the output). */
|
||||
if (derivationIsImpure(derivationType)) {
|
||||
if (!derivationType.isSandboxed()) {
|
||||
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
|
||||
env[i] = getEnv(i).value_or("");
|
||||
}
|
||||
|
|
@ -1156,7 +1183,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig
|
|||
/* A wrapper around LocalStore that only allows building/querying of
|
||||
paths that are in the input closures of the build or were added via
|
||||
recursive Nix calls. */
|
||||
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
|
||||
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore
|
||||
{
|
||||
ref<LocalStore> next;
|
||||
|
||||
|
|
@ -1288,6 +1315,16 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
}
|
||||
|
||||
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
|
||||
{
|
||||
for (auto & result : buildPathsWithResults(paths, buildMode, evalStore))
|
||||
if (!result.success())
|
||||
result.rethrow();
|
||||
}
|
||||
|
||||
std::vector<BuildResult> buildPathsWithResults(
|
||||
const std::vector<DerivedPath> & paths,
|
||||
BuildMode buildMode = bmNormal,
|
||||
std::shared_ptr<Store> evalStore = nullptr) override
|
||||
{
|
||||
assert(!evalStore);
|
||||
|
||||
|
|
@ -1301,26 +1338,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
|
||||
}
|
||||
|
||||
next->buildPaths(paths, buildMode);
|
||||
auto results = next->buildPathsWithResults(paths, buildMode);
|
||||
|
||||
for (auto & path : paths) {
|
||||
auto p = std::get_if<DerivedPath::Built>(&path);
|
||||
if (!p) continue;
|
||||
auto & bfd = *p;
|
||||
auto drv = readDerivation(bfd.drvPath);
|
||||
auto drvHashes = staticOutputHashes(*this, drv);
|
||||
auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
|
||||
for (auto & [outputName, outputPath] : outputs)
|
||||
if (wantOutput(outputName, bfd.outputs)) {
|
||||
newPaths.insert(outputPath);
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
|
||||
auto thisRealisation = next->queryRealisation(
|
||||
DrvOutput{drvHashes.at(outputName), outputName}
|
||||
);
|
||||
assert(thisRealisation);
|
||||
newRealisations.insert(*thisRealisation);
|
||||
}
|
||||
}
|
||||
for (auto & result : results) {
|
||||
for (auto & [outputName, output] : result.builtOutputs) {
|
||||
newPaths.insert(output.outPath);
|
||||
newRealisations.insert(output);
|
||||
}
|
||||
}
|
||||
|
||||
StorePathSet closure;
|
||||
|
|
@ -1329,6 +1353,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
goal.addDependency(path);
|
||||
for (auto & real : Realisation::closure(*next, newRealisations))
|
||||
goal.addedDrvOutputs.insert(real.id);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
|
|
@ -1369,6 +1395,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
|
|||
next->queryMissing(allowed, willBuild, willSubstitute,
|
||||
unknown, downloadSize, narSize);
|
||||
}
|
||||
|
||||
virtual std::optional<std::string> getBuildLog(const StorePath & path) override
|
||||
{ return std::nullopt; }
|
||||
|
||||
virtual void addBuildLog(const StorePath & path, std::string_view log) override
|
||||
{ unsupported("addBuildLog"); }
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -1591,6 +1623,8 @@ void LocalDerivationGoal::runChild()
|
|||
/* Warning: in the child we should absolutely not make any SQLite
|
||||
calls! */
|
||||
|
||||
bool sendException = true;
|
||||
|
||||
try { /* child */
|
||||
|
||||
commonChildInit(builderOut);
|
||||
|
|
@ -1697,7 +1731,7 @@ void LocalDerivationGoal::runChild()
|
|||
/* Fixed-output derivations typically need to access the
|
||||
network, so give them access to /etc/resolv.conf and so
|
||||
on. */
|
||||
if (derivationIsImpure(derivationType)) {
|
||||
if (!derivationType.isSandboxed()) {
|
||||
// Only use nss functions to resolve hosts and
|
||||
// services. Don’t use it for anything else that may
|
||||
// be configured for this system. This limits the
|
||||
|
|
@ -1738,7 +1772,19 @@ void LocalDerivationGoal::runChild()
|
|||
|
||||
for (auto & i : dirsInChroot) {
|
||||
if (i.second.source == "/proc") continue; // backwards compatibility
|
||||
doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
|
||||
|
||||
#if HAVE_EMBEDDED_SANDBOX_SHELL
|
||||
if (i.second.source == "__embedded_sandbox_shell__") {
|
||||
static unsigned char sh[] = {
|
||||
#include "embedded-sandbox-shell.gen.hh"
|
||||
};
|
||||
auto dst = chrootRootDir + i.first;
|
||||
createDirs(dirOf(dst));
|
||||
writeFile(dst, std::string_view((const char *) sh, sizeof(sh)));
|
||||
chmod_(dst, 0555);
|
||||
} else
|
||||
#endif
|
||||
doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
|
||||
}
|
||||
|
||||
/* Bind a new instance of procfs on /proc. */
|
||||
|
|
@ -1954,7 +2000,7 @@ void LocalDerivationGoal::runChild()
|
|||
|
||||
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
|
||||
|
||||
if (derivationIsImpure(derivationType))
|
||||
if (!derivationType.isSandboxed())
|
||||
sandboxProfile += "(import \"sandbox-network.sb\")\n";
|
||||
|
||||
/* Add the output paths we'll use at build-time to the chroot */
|
||||
|
|
@ -2048,6 +2094,8 @@ void LocalDerivationGoal::runChild()
|
|||
/* Indicate that we managed to set up the build environment. */
|
||||
writeFull(STDERR_FILENO, std::string("\2\n"));
|
||||
|
||||
sendException = false;
|
||||
|
||||
/* Execute the program. This should not return. */
|
||||
if (drv->isBuiltin()) {
|
||||
try {
|
||||
|
|
@ -2101,16 +2149,19 @@ void LocalDerivationGoal::runChild()
|
|||
throw SysError("executing '%1%'", drv->builder);
|
||||
|
||||
} catch (Error & e) {
|
||||
writeFull(STDERR_FILENO, "\1\n");
|
||||
FdSink sink(STDERR_FILENO);
|
||||
sink << e;
|
||||
sink.flush();
|
||||
if (sendException) {
|
||||
writeFull(STDERR_FILENO, "\1\n");
|
||||
FdSink sink(STDERR_FILENO);
|
||||
sink << e;
|
||||
sink.flush();
|
||||
} else
|
||||
std::cerr << e.msg();
|
||||
_exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LocalDerivationGoal::registerOutputs()
|
||||
DrvOutputs LocalDerivationGoal::registerOutputs()
|
||||
{
|
||||
/* When using a build hook, the build hook can register the output
|
||||
as valid (by doing `nix-store --import'). If so we don't have
|
||||
|
|
@ -2119,10 +2170,8 @@ void LocalDerivationGoal::registerOutputs()
|
|||
We can only early return when the outputs are known a priori. For
|
||||
floating content-addressed derivations this isn't the case.
|
||||
*/
|
||||
if (hook) {
|
||||
DerivationGoal::registerOutputs();
|
||||
return;
|
||||
}
|
||||
if (hook)
|
||||
return DerivationGoal::registerOutputs();
|
||||
|
||||
std::map<std::string, ValidPathInfo> infos;
|
||||
|
||||
|
|
@ -2163,12 +2212,22 @@ void LocalDerivationGoal::registerOutputs()
|
|||
std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
|
||||
std::map<std::string, struct stat> outputStats;
|
||||
for (auto & [outputName, _] : drv->outputs) {
|
||||
auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
|
||||
auto scratchOutput = get(scratchOutputs, outputName);
|
||||
if (!scratchOutput)
|
||||
throw BuildError(
|
||||
"builder for '%s' has no scratch output for '%s'",
|
||||
worker.store.printStorePath(drvPath), outputName);
|
||||
auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchOutput));
|
||||
|
||||
outputsToSort.insert(outputName);
|
||||
|
||||
/* Updated wanted info to remove the outputs we definitely don't need to register */
|
||||
auto & initialInfo = initialOutputs.at(outputName);
|
||||
auto initialOutput = get(initialOutputs, outputName);
|
||||
if (!initialOutput)
|
||||
throw BuildError(
|
||||
"builder for '%s' has no initial output for '%s'",
|
||||
worker.store.printStorePath(drvPath), outputName);
|
||||
auto & initialInfo = *initialOutput;
|
||||
|
||||
/* Don't register if already valid, and not checking */
|
||||
initialInfo.wanted = buildMode == bmCheck
|
||||
|
|
@ -2223,6 +2282,11 @@ void LocalDerivationGoal::registerOutputs()
|
|||
|
||||
auto sortedOutputNames = topoSort(outputsToSort,
|
||||
{[&](const std::string & name) {
|
||||
auto orifu = get(outputReferencesIfUnregistered, name);
|
||||
if (!orifu)
|
||||
throw BuildError(
|
||||
"no output reference for '%s' in build of '%s'",
|
||||
name, worker.store.printStorePath(drvPath));
|
||||
return std::visit(overloaded {
|
||||
/* Since we'll use the already installed versions of these, we
|
||||
can treat them as leaves and ignore any references they
|
||||
|
|
@ -2237,7 +2301,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
referencedOutputs.insert(o);
|
||||
return referencedOutputs;
|
||||
},
|
||||
}, outputReferencesIfUnregistered.at(name));
|
||||
}, *orifu);
|
||||
}},
|
||||
{[&](const std::string & path, const std::string & parent) {
|
||||
// TODO with more -vvvv also show the temporary paths for manual inspection.
|
||||
|
|
@ -2248,10 +2312,13 @@ void LocalDerivationGoal::registerOutputs()
|
|||
|
||||
std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
|
||||
|
||||
OutputPathMap finalOutputs;
|
||||
|
||||
for (auto & outputName : sortedOutputNames) {
|
||||
auto output = drv->outputs.at(outputName);
|
||||
auto & scratchPath = scratchOutputs.at(outputName);
|
||||
auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
|
||||
auto output = get(drv->outputs, outputName);
|
||||
auto scratchPath = get(scratchOutputs, outputName);
|
||||
assert(output && scratchPath);
|
||||
auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchPath));
|
||||
|
||||
auto finish = [&](StorePath finalStorePath) {
|
||||
/* Store the final path */
|
||||
|
|
@ -2259,10 +2326,13 @@ void LocalDerivationGoal::registerOutputs()
|
|||
/* The rewrite rule will be used in downstream outputs that refer to
|
||||
use. This is why the topological sort is essential to do first
|
||||
before this for loop. */
|
||||
if (scratchPath != finalStorePath)
|
||||
outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
|
||||
if (*scratchPath != finalStorePath)
|
||||
outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() };
|
||||
};
|
||||
|
||||
auto orifu = get(outputReferencesIfUnregistered, outputName);
|
||||
assert(orifu);
|
||||
|
||||
std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
|
||||
[&](const AlreadyRegistered & skippedFinalPath) -> std::optional<StorePathSet> {
|
||||
finish(skippedFinalPath.path);
|
||||
|
|
@ -2271,7 +2341,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
[&](const PerhapsNeedToRegister & r) -> std::optional<StorePathSet> {
|
||||
return r.refs;
|
||||
},
|
||||
}, outputReferencesIfUnregistered.at(outputName));
|
||||
}, *orifu);
|
||||
|
||||
if (!referencesOpt)
|
||||
continue;
|
||||
|
|
@ -2308,25 +2378,29 @@ void LocalDerivationGoal::registerOutputs()
|
|||
for (auto & r : references) {
|
||||
auto name = r.name();
|
||||
auto origHash = std::string { r.hashPart() };
|
||||
if (r == scratchPath)
|
||||
if (r == *scratchPath) {
|
||||
res.first = true;
|
||||
else if (outputRewrites.count(origHash) == 0)
|
||||
res.second.insert(r);
|
||||
else {
|
||||
std::string newRef = outputRewrites.at(origHash);
|
||||
} else if (auto outputRewrite = get(outputRewrites, origHash)) {
|
||||
std::string newRef = *outputRewrite;
|
||||
newRef += '-';
|
||||
newRef += name;
|
||||
res.second.insert(StorePath { newRef });
|
||||
} else {
|
||||
res.second.insert(r);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
};
|
||||
|
||||
auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
|
||||
auto & st = outputStats.at(outputName);
|
||||
auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo {
|
||||
auto st = get(outputStats, outputName);
|
||||
if (!st)
|
||||
throw BuildError(
|
||||
"output path %1% without valid stats info",
|
||||
actualPath);
|
||||
if (outputHash.method == FileIngestionMethod::Flat) {
|
||||
/* The output path should be a regular file without execute permission. */
|
||||
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
|
||||
if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
|
||||
throw BuildError(
|
||||
"output path '%1%' should be a non-executable regular file "
|
||||
"since recursive hashing is not enabled (outputHashMode=flat)",
|
||||
|
|
@ -2334,7 +2408,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
}
|
||||
rewriteOutput();
|
||||
/* FIXME optimize and deduplicate with addToStore */
|
||||
std::string oldHashPart { scratchPath.hashPart() };
|
||||
std::string oldHashPart { scratchPath->hashPart() };
|
||||
HashModuloSink caSink { outputHash.hashType, oldHashPart };
|
||||
switch (outputHash.method) {
|
||||
case FileIngestionMethod::Recursive:
|
||||
|
|
@ -2353,13 +2427,11 @@ void LocalDerivationGoal::registerOutputs()
|
|||
outputPathName(drv->name, outputName),
|
||||
refs.second,
|
||||
refs.first);
|
||||
if (scratchPath != finalPath) {
|
||||
if (*scratchPath != finalPath) {
|
||||
// Also rewrite the output path
|
||||
auto source = sinkToSource([&](Sink & nextSink) {
|
||||
StringSink sink;
|
||||
dumpPath(actualPath, sink);
|
||||
RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink);
|
||||
rsink2(sink.s);
|
||||
dumpPath(actualPath, rsink2);
|
||||
rsink2.flush();
|
||||
});
|
||||
Path tmpPath = actualPath + ".tmp";
|
||||
|
|
@ -2388,14 +2460,15 @@ void LocalDerivationGoal::registerOutputs()
|
|||
};
|
||||
|
||||
ValidPathInfo newInfo = std::visit(overloaded {
|
||||
[&](const DerivationOutputInputAddressed & output) {
|
||||
|
||||
[&](const DerivationOutput::InputAddressed & output) {
|
||||
/* input-addressed case */
|
||||
auto requiredFinalPath = output.path;
|
||||
/* Preemptively add rewrite rule for final hash, as that is
|
||||
what the NAR hash will use rather than normalized-self references */
|
||||
if (scratchPath != requiredFinalPath)
|
||||
if (*scratchPath != requiredFinalPath)
|
||||
outputRewrites.insert_or_assign(
|
||||
std::string { scratchPath.hashPart() },
|
||||
std::string { scratchPath->hashPart() },
|
||||
std::string { requiredFinalPath.hashPart() });
|
||||
rewriteOutput();
|
||||
auto narHashAndSize = hashPath(htSHA256, actualPath);
|
||||
|
|
@ -2407,8 +2480,9 @@ void LocalDerivationGoal::registerOutputs()
|
|||
newInfo0.references.insert(newInfo0.path);
|
||||
return newInfo0;
|
||||
},
|
||||
[&](const DerivationOutputCAFixed & dof) {
|
||||
auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
|
||||
|
||||
[&](const DerivationOutput::CAFixed & dof) {
|
||||
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
|
||||
.method = dof.hash.method,
|
||||
.hashType = dof.hash.hash.type,
|
||||
});
|
||||
|
|
@ -2429,15 +2503,25 @@ void LocalDerivationGoal::registerOutputs()
|
|||
}
|
||||
return newInfo0;
|
||||
},
|
||||
[&](DerivationOutputCAFloating dof) {
|
||||
|
||||
[&](const DerivationOutput::CAFloating & dof) {
|
||||
return newInfoFromCA(dof);
|
||||
},
|
||||
[&](DerivationOutputDeferred) -> ValidPathInfo {
|
||||
|
||||
[&](const DerivationOutput::Deferred &) -> ValidPathInfo {
|
||||
// No derivation should reach that point without having been
|
||||
// rewritten first
|
||||
assert(false);
|
||||
},
|
||||
}, output.output);
|
||||
|
||||
[&](const DerivationOutput::Impure & doi) {
|
||||
return newInfoFromCA(DerivationOutput::CAFloating {
|
||||
.method = doi.method,
|
||||
.hashType = doi.hashType,
|
||||
});
|
||||
},
|
||||
|
||||
}, output->raw());
|
||||
|
||||
/* FIXME: set proper permissions in restorePath() so
|
||||
we don't have to do another traversal. */
|
||||
|
|
@ -2453,7 +2537,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
derivations. */
|
||||
PathLocks dynamicOutputLock;
|
||||
dynamicOutputLock.setDeletion(true);
|
||||
auto optFixedPath = output.path(worker.store, drv->name, outputName);
|
||||
auto optFixedPath = output->path(worker.store, drv->name, outputName);
|
||||
if (!optFixedPath ||
|
||||
worker.store.printStorePath(*optFixedPath) != finalDestPath)
|
||||
{
|
||||
|
|
@ -2519,11 +2603,10 @@ void LocalDerivationGoal::registerOutputs()
|
|||
|
||||
/* For debugging, print out the referenced and unreferenced paths. */
|
||||
for (auto & i : inputPaths) {
|
||||
auto j = references.find(i);
|
||||
if (j == references.end())
|
||||
debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
|
||||
else
|
||||
if (references.count(i))
|
||||
debug("referenced input: '%1%'", worker.store.printStorePath(i));
|
||||
else
|
||||
debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
|
||||
}
|
||||
|
||||
if (curRound == nrRounds) {
|
||||
|
|
@ -2547,11 +2630,12 @@ void LocalDerivationGoal::registerOutputs()
|
|||
}
|
||||
|
||||
if (buildMode == bmCheck) {
|
||||
// In case of FOD mismatches on `--check` an error must be thrown as this is also
|
||||
// a source for non-determinism.
|
||||
/* In case of fixed-output derivations, if there are
|
||||
mismatches on `--check` an error must be thrown as this is
|
||||
also a source for non-determinism. */
|
||||
if (delayedException)
|
||||
std::rethrow_exception(delayedException);
|
||||
return;
|
||||
return assertPathValidity();
|
||||
}
|
||||
|
||||
/* Apply output checks. */
|
||||
|
|
@ -2563,7 +2647,7 @@ void LocalDerivationGoal::registerOutputs()
|
|||
assert(prevInfos.size() == infos.size());
|
||||
for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
|
||||
if (!(*i == *j)) {
|
||||
result.isNonDeterministic = true;
|
||||
buildResult.isNonDeterministic = true;
|
||||
Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
|
||||
bool prevExists = keepPreviousRound && pathExists(prev);
|
||||
hintformat hint = prevExists
|
||||
|
|
@ -2594,14 +2678,13 @@ void LocalDerivationGoal::registerOutputs()
|
|||
Path prev = path + checkSuffix;
|
||||
deletePath(prev);
|
||||
Path dst = path + checkSuffix;
|
||||
if (rename(path.c_str(), dst.c_str()))
|
||||
throw SysError("renaming '%s' to '%s'", path, dst);
|
||||
renameFile(path, dst);
|
||||
}
|
||||
}
|
||||
|
||||
if (curRound < nrRounds) {
|
||||
prevInfos = std::move(infos);
|
||||
return;
|
||||
return {};
|
||||
}
|
||||
|
||||
/* Remove the .check directories if we're done. FIXME: keep them
|
||||
|
|
@ -2636,17 +2719,29 @@ void LocalDerivationGoal::registerOutputs()
|
|||
means it's safe to link the derivation to the output hash. We must do
|
||||
that for floating CA derivations, which otherwise couldn't be cached,
|
||||
but it's fine to do in all cases. */
|
||||
DrvOutputs builtOutputs;
|
||||
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
|
||||
for (auto& [outputName, newInfo] : infos) {
|
||||
auto thisRealisation = Realisation{
|
||||
.id = DrvOutput{initialOutputs.at(outputName).outputHash,
|
||||
outputName},
|
||||
.outPath = newInfo.path};
|
||||
for (auto & [outputName, newInfo] : infos) {
|
||||
auto oldinfo = get(initialOutputs, outputName);
|
||||
assert(oldinfo);
|
||||
auto thisRealisation = Realisation {
|
||||
.id = DrvOutput {
|
||||
oldinfo->outputHash,
|
||||
outputName
|
||||
},
|
||||
.outPath = newInfo.path
|
||||
};
|
||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
|
||||
&& drv->type().isPure())
|
||||
{
|
||||
signRealisation(thisRealisation);
|
||||
worker.store.registerDrvOutput(thisRealisation);
|
||||
}
|
||||
if (wantOutput(outputName, wantedOutputs))
|
||||
builtOutputs.emplace(thisRealisation.id, thisRealisation);
|
||||
}
|
||||
|
||||
return builtOutputs;
|
||||
}
|
||||
|
||||
void LocalDerivationGoal::signRealisation(Realisation & realisation)
|
||||
|
|
@ -2655,7 +2750,7 @@ void LocalDerivationGoal::signRealisation(Realisation & realisation)
|
|||
}
|
||||
|
||||
|
||||
void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
|
||||
void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo> & outputs)
|
||||
{
|
||||
std::map<Path, const ValidPathInfo &> outputsByPath;
|
||||
for (auto & output : outputs)
|
||||
|
|
@ -2727,9 +2822,10 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
|
|||
for (auto & i : *value) {
|
||||
if (worker.store.isStorePath(i))
|
||||
spec.insert(worker.store.parseStorePath(i));
|
||||
else if (finalOutputs.count(i))
|
||||
spec.insert(finalOutputs.at(i));
|
||||
else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
|
||||
else if (auto output = get(outputs, i))
|
||||
spec.insert(output->path);
|
||||
else
|
||||
throw BuildError("derivation contains an illegal reference specifier '%s'", i);
|
||||
}
|
||||
|
||||
auto used = recursive
|
||||
|
|
@ -2768,24 +2864,18 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
|
|||
};
|
||||
|
||||
if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
|
||||
auto outputChecks = structuredAttrs->find("outputChecks");
|
||||
if (outputChecks != structuredAttrs->end()) {
|
||||
auto output = outputChecks->find(outputName);
|
||||
|
||||
if (output != outputChecks->end()) {
|
||||
if (auto outputChecks = get(*structuredAttrs, "outputChecks")) {
|
||||
if (auto output = get(*outputChecks, outputName)) {
|
||||
Checks checks;
|
||||
|
||||
auto maxSize = output->find("maxSize");
|
||||
if (maxSize != output->end())
|
||||
if (auto maxSize = get(*output, "maxSize"))
|
||||
checks.maxSize = maxSize->get<uint64_t>();
|
||||
|
||||
auto maxClosureSize = output->find("maxClosureSize");
|
||||
if (maxClosureSize != output->end())
|
||||
if (auto maxClosureSize = get(*output, "maxClosureSize"))
|
||||
checks.maxClosureSize = maxClosureSize->get<uint64_t>();
|
||||
|
||||
auto get = [&](const std::string & name) -> std::optional<Strings> {
|
||||
auto i = output->find(name);
|
||||
if (i != output->end()) {
|
||||
auto get_ = [&](const std::string & name) -> std::optional<Strings> {
|
||||
if (auto i = get(*output, name)) {
|
||||
Strings res;
|
||||
for (auto j = i->begin(); j != i->end(); ++j) {
|
||||
if (!j->is_string())
|
||||
|
|
@ -2798,10 +2888,10 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
|
|||
return {};
|
||||
};
|
||||
|
||||
checks.allowedReferences = get("allowedReferences");
|
||||
checks.allowedRequisites = get("allowedRequisites");
|
||||
checks.disallowedReferences = get("disallowedReferences");
|
||||
checks.disallowedRequisites = get("disallowedRequisites");
|
||||
checks.allowedReferences = get_("allowedReferences");
|
||||
checks.allowedRequisites = get_("allowedRequisites");
|
||||
checks.disallowedReferences = get_("disallowedReferences");
|
||||
checks.disallowedRequisites = get_("disallowedRequisites");
|
||||
|
||||
applyChecks(checks);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -176,7 +176,7 @@ struct LocalDerivationGoal : public DerivationGoal
|
|||
|
||||
/* Check that the derivation outputs all exist and register them
|
||||
as valid. */
|
||||
void registerOutputs() override;
|
||||
DrvOutputs registerOutputs() override;
|
||||
|
||||
void signRealisation(Realisation &) override;
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
namespace nix {
|
||||
|
||||
PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
: Goal(worker)
|
||||
: Goal(worker, DerivedPath::Opaque { storePath })
|
||||
, storePath(storePath)
|
||||
, repair(repair)
|
||||
, ca(ca)
|
||||
|
|
@ -24,6 +24,20 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
|
|||
}
|
||||
|
||||
|
||||
void PathSubstitutionGoal::done(
|
||||
ExitCode result,
|
||||
BuildResult::Status status,
|
||||
std::optional<std::string> errorMsg)
|
||||
{
|
||||
buildResult.status = status;
|
||||
if (errorMsg) {
|
||||
debug(*errorMsg);
|
||||
buildResult.errorMsg = *errorMsg;
|
||||
}
|
||||
amDone(result);
|
||||
}
|
||||
|
||||
|
||||
void PathSubstitutionGoal::work()
|
||||
{
|
||||
(this->*state)();
|
||||
|
|
@ -38,7 +52,7 @@ void PathSubstitutionGoal::init()
|
|||
|
||||
/* If the path already exists we're done. */
|
||||
if (!repair && worker.store.isValidPath(storePath)) {
|
||||
amDone(ecSuccess);
|
||||
done(ecSuccess, BuildResult::AlreadyValid);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -60,12 +74,14 @@ void PathSubstitutionGoal::tryNext()
|
|||
if (subs.size() == 0) {
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
with it. */
|
||||
debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
|
||||
|
||||
/* Hack: don't indicate failure if there were no substituters.
|
||||
In that case the calling derivation should just do a
|
||||
build. */
|
||||
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
|
||||
done(
|
||||
substituterFailed ? ecFailed : ecNoSubstituters,
|
||||
BuildResult::NoSubstituters,
|
||||
fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)));
|
||||
|
||||
if (substituterFailed) {
|
||||
worker.failedSubstitutions++;
|
||||
|
|
@ -138,7 +154,7 @@ void PathSubstitutionGoal::tryNext()
|
|||
only after we've downloaded the path. */
|
||||
if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
|
||||
{
|
||||
warn("the substitute for '%s' from '%s' is not signed by any of the keys in 'trusted-public-keys'",
|
||||
warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'",
|
||||
worker.store.printStorePath(storePath), sub->getUri());
|
||||
tryNext();
|
||||
return;
|
||||
|
|
@ -162,8 +178,10 @@ void PathSubstitutionGoal::referencesValid()
|
|||
trace("all references realised");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
|
||||
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
|
||||
done(
|
||||
nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
|
||||
BuildResult::DependencyFailed,
|
||||
fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -268,7 +286,7 @@ void PathSubstitutionGoal::finished()
|
|||
|
||||
worker.updateProgress();
|
||||
|
||||
amDone(ecSuccess);
|
||||
done(ecSuccess, BuildResult::Substituted);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,11 @@ struct PathSubstitutionGoal : public Goal
|
|||
/* Content address for recomputing store path */
|
||||
std::optional<ContentAddress> ca;
|
||||
|
||||
void done(
|
||||
ExitCode result,
|
||||
BuildResult::Status status,
|
||||
std::optional<std::string> errorMsg = {});
|
||||
|
||||
public:
|
||||
PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
~PathSubstitutionGoal();
|
||||
|
|
|
|||
|
|
@ -350,7 +350,7 @@ void Worker::waitForInput()
|
|||
become `available'. Note that `available' (i.e., non-blocking)
|
||||
includes EOF. */
|
||||
std::vector<struct pollfd> pollStatus;
|
||||
std::map <int, int> fdToPollStatus;
|
||||
std::map<int, size_t> fdToPollStatus;
|
||||
for (auto & i : children) {
|
||||
for (auto & j : i.fds) {
|
||||
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
|
||||
|
|
@ -380,7 +380,10 @@ void Worker::waitForInput()
|
|||
std::set<int> fds2(j->fds);
|
||||
std::vector<unsigned char> buffer(4096);
|
||||
for (auto & k : fds2) {
|
||||
if (pollStatus.at(fdToPollStatus.at(k)).revents) {
|
||||
const auto fdPollStatusId = get(fdToPollStatus, k);
|
||||
assert(fdPollStatusId);
|
||||
assert(*fdPollStatusId < pollStatus.size());
|
||||
if (pollStatus.at(*fdPollStatusId).revents) {
|
||||
ssize_t rd = ::read(k, buffer.data(), buffer.size());
|
||||
// FIXME: is there a cleaner way to handle pt close
|
||||
// than EIO? Is this even standard?
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue