nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-14 21:11:04 +01:00
Code Activity

Merge remote-tracking branch 'upstream/master' into path-info

Browse source
This commit is contained in:
John Ericson 2021-09-30 22:36:50 +00:00
commit f4f3203aa7
303 changed files with 32784 additions and 3308 deletions
Download patch file Download diff file Expand all files Collapse all files

221
src/libstore/build/derivation-goal.cc
View file

@ -20,6 +20,7 @@
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/wait.h>
#include <netdb.h>
#include <fcntl.h>
#include <termios.h>
@ -142,7 +143,6 @@ void DerivationGoal::work()
(this->*state)();
}
void DerivationGoal::addWantedOutputs(const StringSet & outputs)
{
/* If we already want all outputs, there is nothing to do. */
@ -165,7 +165,7 @@ void DerivationGoal::getDerivation()
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
loadDerivation();
return;
}
@ -188,12 +188,12 @@ void DerivationGoal::loadDerivation()
/* `drvPath' should already be a root, but let's be on the safe
side: if the user forgot to make it a root, we wouldn't want
things being garbage collected while we're busy. */
worker.store.addTempRoot(drvPath);
worker.evalStore.addTempRoot(drvPath);
assert(worker.store.isValidPath(drvPath));
assert(worker.evalStore.isValidPath(drvPath));
/* Get the derivation. */
drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath));
drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath));
haveDerivation();
}
@ -212,8 +212,8 @@ void DerivationGoal::haveDerivation()
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
auto outputHashes = staticOutputHashes(worker.store, *drv);
for (auto &[outputName, outputHash] : outputHashes)
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
for (auto & [outputName, outputHash] : outputHashes)
initialOutputs.insert({
outputName,
InitialOutput{
@ -337,6 +337,15 @@ void DerivationGoal::gaveUpOnSubstitution()
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
/* Copy the input sources from the eval store to the build
store. */
if (&worker.evalStore != &worker.store) {
RealisedPath::Set inputSrcs;
for (auto & i : drv->inputSrcs)
inputSrcs.insert(i);
copyClosure(worker.evalStore, worker.store, inputSrcs);
}
for (auto & i : drv->inputSrcs) {
if (worker.store.isValidPath(i)) continue;
if (!settings.useSubstitutes)
@ -478,8 +487,8 @@ void DerivationGoal::inputsRealised()
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
assert(worker.store.isValidPath(drvPath));
auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
assert(worker.evalStore.isValidPath(drvPath));
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
for (auto & j : wantedDepOutputs) {
if (outputs.count(j) > 0) {
auto optRealizedInput = outputs.at(j);
@ -544,7 +553,7 @@ void DerivationGoal::tryToBuild()
PathSet lockFiles;
/* FIXME: Should lock something like the drv itself so we don't build same
CA drv concurrently */
if (dynamic_cast<LocalStore *>(&worker.store))
if (dynamic_cast<LocalStore *>(&worker.store)) {
/* If we aren't a local store, we might need to use the local store as
a build remote, but that would cause a deadlock. */
/* FIXME: Make it so we can use ourselves as a build remote even if we
@ -552,9 +561,15 @@ void DerivationGoal::tryToBuild()
/* FIXME: find some way to lock for scheduling for the other stores so
a forking daemon with --store still won't farm out redundant builds.
*/
for (auto & i : drv->outputsAndOptPaths(worker.store))
for (auto & i : drv->outputsAndOptPaths(worker.store)) {
if (i.second.second)
lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
else
lockFiles.insert(
worker.store.Store::toRealPath(drvPath) + "." + i.first
);
}
}
if (!outputLocks.lockPaths(lockFiles, "", false)) {
if (!actLock)
@ -738,6 +753,64 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
{
}
void runPostBuildHook(
Store & store,
Logger & logger,
const StorePath & drvPath,
StorePathSet outputPaths
)
{
auto hook = settings.postBuildHook;
if (hook == "")
return;
Activity act(logger, lvlInfo, actPostBuildHook,
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{store.printStorePath(drvPath)});
PushActivity pact(act.id);
std::map<std::string, std::string> hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue());
struct LogSink : Sink {
Activity & act;
std::string currentLine;
LogSink(Activity & act) : act(act) { }
void operator() (std::string_view data) override {
for (auto c : data) {
if (c == '\n') {
flushLine();
} else {
currentLine += c;
}
}
}
void flushLine() {
act.result(resPostBuildLogLine, currentLine);
currentLine.clear();
}
~LogSink() {
if (currentLine != "") {
currentLine += '\n';
flushLine();
}
}
};
LogSink sink(act);
runProgram2({
.program = settings.postBuildHook,
.environment = hookEnvironment,
.standardOut = &sink,
.mergeStderrToStdout = true,
});
}
void DerivationGoal::buildDone()
{
@ -803,57 +876,15 @@ void DerivationGoal::buildDone()
being valid. */
registerOutputs();
if (settings.postBuildHook != "") {
Activity act(*logger, lvlInfo, actPostBuildHook,
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{worker.store.printStorePath(drvPath)});
PushActivity pact(act.id);
StorePathSet outputPaths;
for (auto i : drv->outputs) {
outputPaths.insert(finalOutputs.at(i.first));
}
std::map<std::string, std::string> hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
RunOptions opts(settings.postBuildHook, {});
opts.environment = hookEnvironment;
struct LogSink : Sink {
Activity & act;
std::string currentLine;
LogSink(Activity & act) : act(act) { }
void operator() (std::string_view data) override {
for (auto c : data) {
if (c == '\n') {
flushLine();
} else {
currentLine += c;
}
}
}
void flushLine() {
act.result(resPostBuildLogLine, currentLine);
currentLine.clear();
}
~LogSink() {
if (currentLine != "") {
currentLine += '\n';
flushLine();
}
}
};
LogSink sink(act);
opts.standardOut = &sink;
opts.mergeStderrToStdout = true;
runProgram2(opts);
}
StorePathSet outputPaths;
for (auto & [_, path] : finalOutputs)
outputPaths.insert(path);
runPostBuildHook(
worker.store,
*logger,
drvPath,
outputPaths
);
if (buildMode == bmCheck) {
cleanupPostOutputsRegisteredModeCheck();
@ -909,6 +940,8 @@ void DerivationGoal::resolvedFinished() {
auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs”
auto realWantedOutputs = wantedOutputs;
if (realWantedOutputs.empty())
@ -926,8 +959,10 @@ void DerivationGoal::resolvedFinished() {
auto newRealisation = *realisation;
newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
newRealisation.signatures.clear();
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
outputPaths.insert(realisation->outPath);
} else {
// If we don't have a realisation, then it must mean that something
// failed when building the resolved drv
@ -935,6 +970,13 @@ void DerivationGoal::resolvedFinished() {
}
}
runPostBuildHook(
worker.store,
*logger,
drvPath,
outputPaths
);
// This is potentially a bit fishy in terms of error reporting. Not sure
// how to do it in a cleaner way
amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
@ -967,7 +1009,7 @@ HookReply DerivationGoal::tryBuildHook()
return readLine(worker.hook->fromHook.readSide.get());
} catch (Error & e) {
e.addTrace({}, "while reading the response from the build hook");
throw e;
throw;
}
}();
if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
@ -1013,7 +1055,7 @@ HookReply DerivationGoal::tryBuildHook()
machineName = readLine(hook->fromHook.readSide.get());
} catch (Error & e) {
e.addTrace({}, "while reading the machine name from the build hook");
throw e;
throw;
}
/* Tell the hook all the inputs that have to be copied to the
@ -1047,42 +1089,6 @@ HookReply DerivationGoal::tryBuildHook()
}
StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
{
StorePathSet paths;
for (auto & storePath : storePaths) {
if (!inputPaths.count(storePath))
throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
worker.store.computeFSClosure({storePath}, paths);
}
/* If there are derivations in the graph, then include their
outputs as well. This is useful if you want to do things
like passing all build-time dependencies of some path to a
derivation that builds a NixOS DVD image. */
auto paths2 = paths;
for (auto & j : paths2) {
if (j.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(j);
for (auto & k : drv.outputsAndOptPaths(worker.store)) {
if (!k.second.second)
/* FIXME: I am confused why we are calling
`computeFSClosure` on the output path, rather than
derivation itself. That doesn't seem right to me, so I
won't try to implemented this for CA derivations. */
throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
worker.store.computeFSClosure(*k.second.second, paths);
}
}
}
return paths;
}
void DerivationGoal::registerOutputs()
{
/* When using a build hook, the build hook can register the output
@ -1268,12 +1274,23 @@ void DerivationGoal::checkPathValidity()
};
}
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (auto real = worker.store.queryRealisation(
DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) {
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = {
.path = real->outPath,
.status = PathStatus::Valid,
};
} else if (info.known && info.known->status == PathStatus::Valid) {
// We know the output because it' a static output of the
// derivation, and the output path is valid, but we don't have
// its realisation stored (probably because it has been built
// without the `ca-derivations` experimental flag)
worker.store.registerDrvOutput(
Realisation{
drvOutput,
info.known->path,
}
);
}
}
}

27
src/libstore/build/drv-output-substitution-goal.cc
View file

@ -17,6 +17,13 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker
void DrvOutputSubstitutionGoal::init()
{
trace("init");
/* If the derivation already exists, we’re done */
if (worker.store.queryRealisation(id)) {
amDone(ecSuccess);
return;
}
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
tryNext();
}
@ -53,6 +60,26 @@ void DrvOutputSubstitutionGoal::tryNext()
return;
}
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
if (depId != id) {
if (auto localOutputInfo = worker.store.queryRealisation(depId);
localOutputInfo && localOutputInfo->outPath != depPath) {
warn(
"substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
"Local: %s\n"
"Remote: %s",
sub->getUri(),
depId.to_string(),
worker.store.printStorePath(localOutputInfo->outPath),
worker.store.printStorePath(depPath)
);
tryNext();
return;
}
addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
}
}
addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
if (waitees.empty()) outPathValid();

10
src/libstore/build/entry-points.cc
View file

@ -6,9 +6,9 @@
namespace nix {
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode)
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
Worker worker(*this);
Worker worker(*this, evalStore ? *evalStore : *this);
Goals goals;
for (auto & br : reqs) {
@ -51,7 +51,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
Worker worker(*this);
Worker worker(*this, *this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
BuildResult result;
@ -93,7 +93,7 @@ void Store::ensurePath(const StorePath & path)
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
Worker worker(*this);
Worker worker(*this, *this);
GoalPtr goal = worker.makePathSubstitutionGoal(path);
Goals goals = {goal};
@ -111,7 +111,7 @@ void Store::ensurePath(const StorePath & path)
void LocalStore::repairPath(const StorePath & path)
{
Worker worker(*this);
Worker worker(*this, *this);
GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
Goals goals = {goal};

15
src/libstore/build/goal.cc
View file

@ -13,11 +13,9 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
{
// FIXME: necessary?
// FIXME: O(n)
for (auto & i : goals)
if (i.lock() == p) return;
goals.push_back(p);
if (goals.find(p) != goals.end())
return;
goals.insert(p);
}
@ -46,10 +44,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (auto & goal : waitees) {
WeakGoals waiters2;
for (auto & j : goal->waiters)
if (j.lock() != shared_from_this()) waiters2.push_back(j);
goal->waiters = waiters2;
goal->waiters.extract(shared_from_this());
}
waitees.clear();
@ -78,6 +73,8 @@ void Goal::amDone(ExitCode result, std::optional<Error> ex)
}
waiters.clear();
worker.removeGoal(shared_from_this());
cleanup();
}

4
src/libstore/build/goal.hh
View file

@ -19,7 +19,7 @@ struct CompareGoalPtrs {
/* Set of goals. */
typedef set<GoalPtr, CompareGoalPtrs> Goals;
typedef list<WeakGoalPtr> WeakGoals;
typedef set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
/* A map of paths to goals (and the other way around). */
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
@ -100,6 +100,8 @@ struct Goal : public std::enable_shared_from_this<Goal>
virtual string key() = 0;
void amDone(ExitCode result, std::optional<Error> ex = {});
virtual void cleanup() { }
};
void addToWeakGoals(WeakGoals & goals, GoalPtr p);

245
src/libstore/build/local-derivation-goal.cc
View file

@ -17,16 +17,14 @@
#include <regex>
#include <queue>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <netdb.h>
#include <fcntl.h>
#include <termios.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/utsname.h>
#include <sys/resource.h>
#include <sys/socket.h>
#if HAVE_STATVFS
#include <sys/statvfs.h>
@ -34,7 +32,6 @@
/* Includes required for chroot support. */
#if __linux__
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <netinet/ip.h>
@ -70,12 +67,14 @@ void handleDiffHook(
auto diffHook = settings.diffHook;
if (diffHook != "" && settings.runDiffHook) {
try {
RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
diffHookOptions.searchPath = true;
diffHookOptions.uid = uid;
diffHookOptions.gid = gid;
diffHookOptions.chdir = "/";
auto diffRes = runProgram(diffHookOptions);
auto diffRes = runProgram(RunOptions {
.program = diffHook,
.searchPath = true,
.args = {tryA, tryB, drvPath, tmpDir},
.uid = uid,
.gid = gid,
.chdir = "/"
});
if (!statusOk(diffRes.first))
throw ExecError(diffRes.first,
"diff-hook program '%1%' %2%",
@ -153,6 +152,7 @@ void LocalDerivationGoal::killChild()
void LocalDerivationGoal::tryLocalBuild() {
unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= settings.maxBuildJobs) {
state = &DerivationGoal::tryToBuild;
worker.waitForBuildSlot(shared_from_this());
outputLocks.unlock();
return;
@ -291,7 +291,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
auto & localStore = getLocalStore();
uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st;
if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 &&
(uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
if (statvfs(tmpDir.c_str(), &st) == 0 &&
@ -343,23 +343,6 @@ int childEntry(void * arg)
}
static std::once_flag dns_resolve_flag;
static void preloadNSS() {
/* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
load its lookup libraries in the parent before any child gets a chance to. */
std::call_once(dns_resolve_flag, []() {
struct addrinfo *res = NULL;
if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
if (res) freeaddrinfo(res);
}
});
}
static void linkOrCopy(const Path & from, const Path & to)
{
if (link(from.c_str(), to.c_str()) == -1) {
@ -388,9 +371,6 @@ void LocalDerivationGoal::startBuilder()
settings.thisSystem,
concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
if (drv->isBuiltin())
preloadNSS();
#if __APPLE__
additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
#endif
@ -416,7 +396,7 @@ void LocalDerivationGoal::startBuilder()
}
auto & localStore = getLocalStore();
if (localStore.storeDir != localStore.realStoreDir) {
if (localStore.storeDir != localStore.realStoreDir.get()) {
#if __linux__
useChroot = true;
#else
@ -517,7 +497,7 @@ void LocalDerivationGoal::startBuilder()
/* Write closure info to <fileName>. */
writeFile(tmpDir + "/" + fileName,
worker.store.makeValidityRegistration(
exportReferences({storePath}), false, false));
worker.store.exportReferences({storePath}, inputPaths), false, false));
}
}
@ -581,7 +561,9 @@ void LocalDerivationGoal::startBuilder()
throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
worker.store.printStorePath(drvPath), i);
dirsInChroot[i] = i;
/* Allow files in __impureHostDeps to be missing; e.g.
macOS 11+ has no /usr/lib/libSystem*.dylib */
dirsInChroot[i] = {i, true};
}
#if __linux__
@ -956,9 +938,12 @@ void LocalDerivationGoal::startBuilder()
try {
return readLine(builderOut.readSide.get());
} catch (Error & e) {
e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)",
auto status = pid.wait();
e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)",
worker.store.printStorePath(drvPath),
statusToString(status),
concatStringsSep("|", msgs));
throw e;
throw;
}
}();
if (string(msg, 0, 1) == "\2") break;
@ -966,7 +951,7 @@ void LocalDerivationGoal::startBuilder()
FdSource source(builderOut.readSide.get());
auto ex = readError(source);
ex.addTrace({}, "while setting up the build environment");
throw ex;
throw;
}
debug("sandbox setup: " + msg);
msgs.push_back(std::move(msg));
@ -1081,113 +1066,28 @@ void LocalDerivationGoal::initEnv()
}
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
void LocalDerivationGoal::writeStructuredAttrs()
{
auto structuredAttrs = parsedDrv->getStructuredAttrs();
if (!structuredAttrs) return;
if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) {
auto json = structAttrsJson.value();
nlohmann::json rewritten;
for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
/* The placeholder must have a rewrite, so we use it to cover both the
cases where we know or don't know the output path ahead of time. */
rewritten[i] = rewriteStrings(v, inputRewrites);
}
auto json = *structuredAttrs;
json["outputs"] = rewritten;
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
for (auto & i : drv->outputs) {
/* The placeholder must have a rewrite, so we use it to cover both the
cases where we know or don't know the output path ahead of time. */
outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
auto jsonSh = writeStructuredAttrsShell(json);
writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
chownToBuilder(tmpDir + "/.attrs.sh");
env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh";
writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
chownToBuilder(tmpDir + "/.attrs.json");
env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json";
}
json["outputs"] = outputs;
/* Handle exportReferencesGraph. */
auto e = json.find("exportReferencesGraph");
if (e != json.end() && e->is_object()) {
for (auto i = e->begin(); i != e->end(); ++i) {
std::ostringstream str;
{
JSONPlaceholder jsonRoot(str, true);
StorePathSet storePaths;
for (auto & p : *i)
storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
worker.store.pathInfoToJSON(jsonRoot,
exportReferences(storePaths), false, true);
}
json[i.key()] = nlohmann::json::parse(str.str()); // urgh
}
}
writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
chownToBuilder(tmpDir + "/.attrs.json");
/* As a convenience to bash scripts, write a shell file that
maps all attributes that are representable in bash -
namely, strings, integers, nulls, Booleans, and arrays and
objects consisting entirely of those values. (So nested
arrays or objects are not supported.) */
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
if (value.is_string())
return shellEscape(value);
if (value.is_number()) {
auto f = value.get<float>();
if (std::ceil(f) == f)
return std::to_string(value.get<int>());
}
if (value.is_null())
return std::string("''");
if (value.is_boolean())
return value.get<bool>() ? std::string("1") : std::string("");
return {};
};
std::string jsonSh;
for (auto i = json.begin(); i != json.end(); ++i) {
if (!std::regex_match(i.key(), shVarName)) continue;
auto & value = i.value();
auto s = handleSimpleType(value);
if (s)
jsonSh += fmt("declare %s=%s\n", i.key(), *s);
else if (value.is_array()) {
std::string s2;
bool good = true;
for (auto i = value.begin(); i != value.end(); ++i) {
auto s3 = handleSimpleType(i.value());
if (!s3) { good = false; break; }
s2 += *s3; s2 += ' ';
}
if (good)
jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
}
else if (value.is_object()) {
std::string s2;
bool good = true;
for (auto i = value.begin(); i != value.end(); ++i) {
auto s3 = handleSimpleType(i.value());
if (!s3) { good = false; break; }
s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
}
if (good)
jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
}
}
writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
chownToBuilder(tmpDir + "/.attrs.sh");
}
@ -1330,13 +1230,20 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
// XXX: This should probably be allowed if the realisation corresponds to
// an allowed derivation
{ throw Error("queryRealisation"); }
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
{
if (!goal.isAllowed(id))
throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
return next->queryRealisation(id);
}
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{
assert(!evalStore);
if (buildMode != bmNormal) throw Error("unsupported build mode");
StorePathSet newPaths;
std::set<Realisation> newRealisations;
for (auto & req : paths) {
if (!goal.isAllowed(req))
@ -1349,16 +1256,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
auto p = std::get_if<DerivedPath::Built>(&path);
if (!p) continue;
auto & bfd = *p;
auto drv = readDerivation(bfd.drvPath);
auto drvHashes = staticOutputHashes(*this, drv);
auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
for (auto & [outputName, outputPath] : outputs)
if (wantOutput(outputName, bfd.outputs))
if (wantOutput(outputName, bfd.outputs)) {
newPaths.insert(outputPath);
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
auto thisRealisation = next->queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName}
);
assert(thisRealisation);
newRealisations.insert(*thisRealisation);
}
}
}
StorePathSet closure;
next->computeFSClosure(newPaths, closure);
for (auto & path : closure)
goal.addDependency(path);
for (auto & real : Realisation::closure(*next, newRealisations))
goal.addedDrvOutputs.insert(real.id);
}
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@ -1734,7 +1653,7 @@ void LocalDerivationGoal::runChild()
/* N.B. it is realistic that these paths might not exist. It
happens when testing Nix building fixed-output derivations
within a pure derivation. */
for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" })
for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
if (pathExists(path))
ss.push_back(path);
}
@ -1916,7 +1835,7 @@ void LocalDerivationGoal::runChild()
/* Fill in the arguments. */
Strings args;
const char *builder = "invalid";
std::string builder = "invalid";
if (drv->isBuiltin()) {
;
@ -2042,13 +1961,13 @@ void LocalDerivationGoal::runChild()
}
args.push_back(drv->builder);
} else {
builder = drv->builder.c_str();
builder = drv->builder;
args.push_back(std::string(baseNameOf(drv->builder)));
}
}
#else
else {
builder = drv->builder.c_str();
builder = drv->builder;
args.push_back(std::string(baseNameOf(drv->builder)));
}
#endif
@ -2104,9 +2023,9 @@ void LocalDerivationGoal::runChild()
posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
}
posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
#else
execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
#endif
throw SysError("executing '%1%'", drv->builder);
@ -2298,10 +2217,6 @@ void LocalDerivationGoal::registerOutputs()
sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
StringSource source(*sink.s);
restorePath(actualPath, source);
/* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */
canonicalisePathMetaData(actualPath, -1, inodesSeen);
}
};
@ -2353,9 +2268,6 @@ void LocalDerivationGoal::registerOutputs()
break;
}
auto got = caSink.finish().first;
HashModuloSink narSink { htSHA256, oldHashPart };
dumpPath(actualPath, narSink);
auto narHashAndSize = narSink.finish();
ValidPathInfo newInfo0 {
worker.store,
{
@ -2368,9 +2280,8 @@ void LocalDerivationGoal::registerOutputs()
rewriteRefs(),
},
},
narHashAndSize.first,
Hash::dummy,
};
newInfo0.narSize = narHashAndSize.second;
if (scratchPath != newInfo0.path) {
// Also rewrite the output path
auto source = sinkToSource([&](Sink & nextSink) {
@ -2386,6 +2297,10 @@ void LocalDerivationGoal::registerOutputs()
movePath(tmpPath, actualPath);
}
HashResult narHashAndSize = hashPath(htSHA256, actualPath);
newInfo0.narHash = narHashAndSize.first;
newInfo0.narSize = narHashAndSize.second;
assert(newInfo0.ca);
return newInfo0;
};
@ -2443,6 +2358,10 @@ void LocalDerivationGoal::registerOutputs()
},
}, output.output);
/* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */
canonicalisePathMetaData(actualPath, -1, inodesSeen);
/* Calculate where we'll move the output files. In the checking case we
will leave leave them where they are, for now, rather than move to
their usual "final destination" */
@ -2452,6 +2371,7 @@ void LocalDerivationGoal::registerOutputs()
floating CA derivations and hash-mismatching fixed-output
derivations. */
PathLocks dynamicOutputLock;
dynamicOutputLock.setDeletion(true);
auto optFixedPath = output.path(worker.store, drv->name, outputName);
if (!optFixedPath ||
worker.store.printStorePath(*optFixedPath) != finalDestPath)
@ -2475,6 +2395,7 @@ void LocalDerivationGoal::registerOutputs()
assert(newInfo.ca);
} else {
auto destPath = worker.store.toRealPath(finalDestPath);
deletePath(destPath);
movePath(actualPath, destPath);
actualPath = destPath;
}
@ -2544,7 +2465,13 @@ void LocalDerivationGoal::registerOutputs()
infos.emplace(outputName, std::move(newInfo));
}
if (buildMode == bmCheck) return;
if (buildMode == bmCheck) {
// In case of FOD mismatches on `--check` an error must be thrown as this is also
// a source for non-determinism.
if (delayedException)
std::rethrow_exception(delayedException);
return;
}
/* Apply output checks. */
checkOutputs(infos);

8
src/libstore/build/local-derivation-goal.hh
View file

@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal
/* Paths that were added via recursive Nix calls. */
StorePathSet addedPaths;
/* Realisations that were added via recursive Nix calls. */
std::set<DrvOutput> addedDrvOutputs;
/* Recursive Nix calls are only allowed to build or realize paths
in the original input closure or added via a recursive Nix call
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
@ -116,6 +119,11 @@ struct LocalDerivationGoal : public DerivationGoal
{
return inputPaths.count(path) || addedPaths.count(path);
}
bool isAllowed(const DrvOutput & id)
{
return addedDrvOutputs.count(id);
}
bool isAllowed(const DerivedPath & req);
friend struct RestrictedStore;

33
src/libstore/build/substitution-goal.cc
View file

@ -20,15 +20,7 @@ PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker &
PathSubstitutionGoal::~PathSubstitutionGoal()
{
try {
if (thr.joinable()) {
// FIXME: signal worker thread to quit.
thr.join();
worker.childTerminated(this);
}
} catch (...) {
ignoreException();
}
cleanup();
}
@ -63,6 +55,8 @@ void PathSubstitutionGoal::tryNext()
{
trace("trying next substituter");
cleanup();
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
@ -206,12 +200,12 @@ void PathSubstitutionGoal::tryToRun()
thr = std::thread([this]() {
try {
/* Wake up the worker loop when we're done. */
Finally updateStats([this]() { outPipe.writeSide = -1; });
Finally updateStats([this]() { outPipe.writeSide.close(); });
Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
copyStorePath(*sub, worker.store,
subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
@ -289,4 +283,21 @@ void PathSubstitutionGoal::handleEOF(int fd)
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}
void PathSubstitutionGoal::cleanup()
{
try {
if (thr.joinable()) {
// FIXME: signal worker thread to quit.
thr.join();
worker.childTerminated(this);
}
outPipe.close();
} catch (...) {
ignoreException();
}
}
}

4
src/libstore/build/substitution-goal.hh
View file

@ -14,7 +14,7 @@ struct PathSubstitutionGoal : public Goal
StorePath storePath;
/* The path the substituter refers to the path as. This will be
* different when the stores have different names. */
different when the stores have different names. */
std::optional<StorePath> subPath;
/* The remaining substituters. */
@ -79,6 +79,8 @@ public:
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleEOF(int fd) override;
void cleanup() override;
};
}

4
src/libstore/build/worker.cc
View file

@ -9,11 +9,12 @@
namespace nix {
Worker::Worker(Store & store)
Worker::Worker(Store & store, Store & evalStore)
: act(*logger, actRealise)
, actDerivations(*logger, actBuilds)
, actSubstitutions(*logger, actCopyPaths)
, store(store)
, evalStore(evalStore)
{
/* Debugging: prevent recursive workers. */
nrLocalBuilds = 0;
@ -128,6 +129,7 @@ void Worker::removeGoal(GoalPtr goal)
nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
else
assert(false);
if (topGoals.find(goal) != topGoals.end()) {
topGoals.erase(goal);
/* If a top-level goal failed, then kill all other goals

3
src/libstore/build/worker.hh
View file

@ -110,6 +110,7 @@ public:
bool checkMismatch;
Store & store;
Store & evalStore;
std::unique_ptr<HookInstance> hook;
@ -131,7 +132,7 @@ public:
it answers with "decline-permanently", we don't try again. */
bool tryBuildHook = true;
Worker(Store & store);
Worker(Store & store, Store & evalStore);
~Worker();
/* Make a goal (with caching). */