nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-20 01:09:37 +01:00
Code Activity

Merge branch 'master' into path-info

Browse source
This commit is contained in:
John Ericson 2021-04-05 18:29:52 -04:00
commit e12308dd63
95 changed files with 1130 additions and 423 deletions
Download patch file Download diff file Expand all files Collapse all files

43
src/libstore/build/derivation-goal.cc
View file

@ -170,7 +170,7 @@ void DerivationGoal::getDerivation()
return;
}
addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath)));
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
state = &DerivationGoal::loadDerivation;
}
@ -246,17 +246,22 @@ void DerivationGoal::haveDerivation()
through substitutes. If that doesn't work, we'll build
them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
for (auto & [_, status] : initialOutputs) {
for (auto & [outputName, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known) {
warn("do not know how to query for unknown floating content-addressed derivation output yet");
/* Nothing to wait for; tail call */
return DerivationGoal::gaveUpOnSubstitution();
}
addWaitee(upcast_goal(worker.makeSubstitutionGoal(
status.known->path,
buildMode == bmRepair ? Repair : NoRepair,
getDerivationCA(*drv))));
if (!status.known)
addWaitee(
upcast_goal(
worker.makeDrvOutputSubstitutionGoal(
DrvOutput{status.outputHash, outputName},
buildMode == bmRepair ? Repair : NoRepair
)
)
);
else
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
status.known->path,
buildMode == bmRepair ? Repair : NoRepair,
getDerivationCA(*drv))));
}
if (waitees.empty()) /* to prevent hang (no wake-up event) */
@ -337,7 +342,7 @@ void DerivationGoal::gaveUpOnSubstitution()
if (!settings.useSubstitutes)
throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
addWaitee(upcast_goal(worker.makeSubstitutionGoal(i)));
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i)));
}
if (waitees.empty()) /* to prevent hang (no wake-up event) */
@ -388,7 +393,7 @@ void DerivationGoal::repairClosure()
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
auto drvPath2 = outputsToDrv.find(i);
if (drvPath2 == outputsToDrv.end())
addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair)));
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else
addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
}
@ -920,6 +925,8 @@ void DerivationGoal::resolvedFinished() {
if (realisation) {
auto newRealisation = *realisation;
newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
newRealisation.signatures.clear();
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
} else {
// If we don't have a realisation, then it must mean that something
@ -1243,9 +1250,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
void DerivationGoal::checkPathValidity()
{
bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs;
for (auto & i : queryPartialDerivationOutputMap()) {
InitialOutput & info = initialOutputs.at(i.first);
info.wanted = wantOutput(i.first, wantedOutputs);
if (info.wanted)
wantedOutputsLeft.erase(i.first);
if (i.second) {
auto outputPath = *i.second;
info.known = {
@ -1267,6 +1277,13 @@ void DerivationGoal::checkPathValidity()
}
}
}
// If we requested all the outputs via the empty set, we are always fine.
// If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid.
if (!wantedOutputsLeft.empty())
throw Error("derivation '%s' does not have wanted outputs %s",
worker.store.printStorePath(drvPath),
concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
}

3
src/libstore/build/derivation-goal.hh
View file

@ -180,6 +180,9 @@ struct DerivationGoal : public Goal
/* Open a log file and a pipe to it. */
Path openLogFile();
/* Sign the newly built realisation if the store allows it */
virtual void signRealisation(Realisation&) {}
/* Close the log file. */
void closeLogFile();

95
src/libstore/build/drv-output-substitution-goal.cc Normal file
View file

@ -0,0 +1,95 @@
#include "drv-output-substitution-goal.hh"
#include "worker.hh"
#include "substitution-goal.hh"
namespace nix {
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, id(id)
{
state = &DrvOutputSubstitutionGoal::init;
name = fmt("substitution of '%s'", id.to_string());
trace("created");
}
void DrvOutputSubstitutionGoal::init()
{
trace("init");
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
tryNext();
}
void DrvOutputSubstitutionGoal::tryNext()
{
trace("Trying next substituter");
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
/* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a
build. */
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
if (substituterFailed) {
worker.failedSubstitutions++;
worker.updateProgress();
}
return;
}
auto sub = subs.front();
subs.pop_front();
// FIXME: Make async
outputInfo = sub->queryRealisation(id);
if (!outputInfo) {
tryNext();
return;
}
addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
if (waitees.empty()) outPathValid();
else state = &DrvOutputSubstitutionGoal::outPathValid;
}
void DrvOutputSubstitutionGoal::outPathValid()
{
assert(outputInfo);
trace("Output path substituted");
if (nrFailed > 0) {
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
return;
}
worker.store.registerDrvOutput(*outputInfo);
finished();
}
void DrvOutputSubstitutionGoal::finished()
{
trace("finished");
amDone(ecSuccess);
}
string DrvOutputSubstitutionGoal::key()
{
/* "a$" ensures substitution goals happen before derivation
goals. */
return "a$" + std::string(id.to_string());
}
void DrvOutputSubstitutionGoal::work()
{
(this->*state)();
}
}

50
src/libstore/build/drv-output-substitution-goal.hh Normal file
View file

@ -0,0 +1,50 @@
#pragma once
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
namespace nix {
class Worker;
// Substitution of a derivation output.
// This is done in three steps:
// 1. Fetch the output info from a substituter
// 2. Substitute the corresponding output path
// 3. Register the output info
class DrvOutputSubstitutionGoal : public Goal {
private:
// The drv output we're trying to substitue
DrvOutput id;
// The realisation corresponding to the given output id.
// Will be filled once we can get it.
std::optional<Realisation> outputInfo;
/* The remaining substituters. */
std::list<ref<Store>> subs;
/* Whether a substituter failed. */
bool substituterFailed = false;
public:
DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
typedef void (DrvOutputSubstitutionGoal::*GoalState)();
GoalState state;
void init();
void tryNext();
void outPathValid();
void finished();
void timedOut(Error && ex) override { abort(); };
string key() override;
void work() override;
};
}

8
src/libstore/build/entry-points.cc
View file

@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, Build
if (path.path.isDerivation())
goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
else
goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
}
worker.run(goals);
@ -31,7 +31,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, Build
}
if (i->exitCode != Goal::ecSuccess) {
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
else if (auto i2 = dynamic_cast<SubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
}
}
@ -90,7 +90,7 @@ void Store::ensurePath(const StorePath & path)
if (isValidPath(path)) return;
Worker worker(*this);
GoalPtr goal = worker.makeSubstitutionGoal(path);
GoalPtr goal = worker.makePathSubstitutionGoal(path);
Goals goals = {goal};
worker.run(goals);
@ -108,7 +108,7 @@ void Store::ensurePath(const StorePath & path)
void LocalStore::repairPath(const StorePath & path)
{
Worker worker(*this);
GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
Goals goals = {goal};
worker.run(goals);

33
src/libstore/build/local-derivation-goal.cc
View file

@ -287,7 +287,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
So instead, check if the disk is (nearly) full now. If
so, we don't mark this build as a permanent failure. */
#if HAVE_STATVFS
{
{
auto & localStore = getLocalStore();
uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st;
@ -297,7 +297,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
if (statvfs(tmpDir.c_str(), &st) == 0 &&
(uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
}
}
#endif
deleteTmpDir(false);
@ -1703,18 +1703,18 @@ void LocalDerivationGoal::runChild()
network, so give them access to /etc/resolv.conf and so
on. */
if (derivationIsImpure(derivationType)) {
ss.push_back("/etc/resolv.conf");
// Only use nss functions to resolve hosts and
// services. Don’t use it for anything else that may
// be configured for this system. This limits the
// potential impurities introduced in fixed-outputs.
writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
ss.push_back("/etc/services");
ss.push_back("/etc/hosts");
if (pathExists("/var/run/nscd/socket"))
ss.push_back("/var/run/nscd/socket");
/* N.B. it is realistic that these paths might not exist. It
happens when testing Nix building fixed-output derivations
within a pure derivation. */
for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" })
if (pathExists(path))
ss.push_back(path);
}
for (auto & i : ss) dirsInChroot.emplace(i, i);
@ -2608,13 +2608,22 @@ void LocalDerivationGoal::registerOutputs()
but it's fine to do in all cases. */
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
for (auto& [outputName, newInfo] : infos)
worker.store.registerDrvOutput(Realisation{
.id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName},
.outPath = newInfo.path});
for (auto& [outputName, newInfo] : infos) {
auto thisRealisation = Realisation{
.id = DrvOutput{initialOutputs.at(outputName).outputHash,
outputName},
.outPath = newInfo.path};
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}
}
}
void LocalDerivationGoal::signRealisation(Realisation & realisation)
{
getLocalStore().signRealisation(realisation);
}
void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
{

2
src/libstore/build/local-derivation-goal.hh
View file

@ -161,6 +161,8 @@ struct LocalDerivationGoal : public DerivationGoal
as valid. */
void registerOutputs() override;
void signRealisation(Realisation &) override;
/* Check that an output meets the requirements specified by the
'outputChecks' attribute (or the legacy
'{allowed,disallowed}{References,Requisites}' attributes). */

34
src/libstore/build/substitution-goal.cc
View file

@ -5,20 +5,20 @@
namespace nix {
SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
, ca(ca)
{
state = &SubstitutionGoal::init;
state = &PathSubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
trace("created");
maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
}
SubstitutionGoal::~SubstitutionGoal()
PathSubstitutionGoal::~PathSubstitutionGoal()
{
try {
if (thr.joinable()) {
@ -32,13 +32,13 @@ SubstitutionGoal::~SubstitutionGoal()
}
void SubstitutionGoal::work()
void PathSubstitutionGoal::work()
{
(this->*state)();
}
void SubstitutionGoal::init()
void PathSubstitutionGoal::init()
{
trace("init");
@ -59,7 +59,7 @@ void SubstitutionGoal::init()
}
void SubstitutionGoal::tryNext()
void PathSubstitutionGoal::tryNext()
{
trace("trying next substituter");
@ -145,7 +145,7 @@ void SubstitutionGoal::tryNext()
/* Bail out early if this substituter lacks a valid
signature. LocalStore::addToStore() also checks for this, but
only after we've downloaded the path. */
if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info))
if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
{
warn("substituter '%s' does not have a valid signature for path '%s'",
sub->getUri(), worker.store.printStorePath(storePath));
@ -156,16 +156,16 @@ void SubstitutionGoal::tryNext()
/* To maintain the closure invariant, we first have to realise the
paths referenced by this one. */
for (auto & i : info->references)
addWaitee(worker.makeSubstitutionGoal(i));
addWaitee(worker.makePathSubstitutionGoal(i));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
referencesValid();
else
state = &SubstitutionGoal::referencesValid;
state = &PathSubstitutionGoal::referencesValid;
}
void SubstitutionGoal::referencesValid()
void PathSubstitutionGoal::referencesValid()
{
trace("all references realised");
@ -178,12 +178,12 @@ void SubstitutionGoal::referencesValid()
for (auto & i : info->references)
assert(worker.store.isValidPath(i));
state = &SubstitutionGoal::tryToRun;
state = &PathSubstitutionGoal::tryToRun;
worker.wakeUp(shared_from_this());
}
void SubstitutionGoal::tryToRun()
void PathSubstitutionGoal::tryToRun()
{
trace("trying to run");
@ -222,11 +222,11 @@ void SubstitutionGoal::tryToRun()
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
state = &SubstitutionGoal::finished;
state = &PathSubstitutionGoal::finished;
}
void SubstitutionGoal::finished()
void PathSubstitutionGoal::finished()
{
trace("substitute finished");
@ -250,7 +250,7 @@ void SubstitutionGoal::finished()
}
/* Try the next substitute. */
state = &SubstitutionGoal::tryNext;
state = &PathSubstitutionGoal::tryNext;
worker.wakeUp(shared_from_this());
return;
}
@ -279,12 +279,12 @@ void SubstitutionGoal::finished()
}
void SubstitutionGoal::handleChildOutput(int fd, const string & data)
void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
{
}
void SubstitutionGoal::handleEOF(int fd)
void PathSubstitutionGoal::handleEOF(int fd)
{
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}

9
src/libstore/build/substitution-goal.hh
View file

@ -8,7 +8,7 @@ namespace nix {
class Worker;
struct SubstitutionGoal : public Goal
struct PathSubstitutionGoal : public Goal
{
/* The store path that should be realised through a substitute. */
StorePath storePath;
@ -47,14 +47,15 @@ struct SubstitutionGoal : public Goal
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
typedef void (SubstitutionGoal::*GoalState)();
typedef void (PathSubstitutionGoal::*GoalState)();
GoalState state;
/* Content address for recomputing store path */
std::optional<ContentAddress> ca;
SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~SubstitutionGoal();
public:
PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~PathSubstitutionGoal();
void timedOut(Error && ex) override { abort(); };

34
src/libstore/build/worker.cc
View file

@ -1,6 +1,7 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "drv-output-substitution-goal.hh"
#include "local-derivation-goal.hh"
#include "hook-instance.hh"
@ -78,20 +79,32 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
}
std::shared_ptr<SubstitutionGoal> Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
std::weak_ptr<SubstitutionGoal> & goal_weak = substitutionGoals[path];
std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
auto goal = goal_weak.lock(); // FIXME
if (!goal) {
goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
goal = std::make_shared<PathSubstitutionGoal>(path, *this, repair, ca);
goal_weak = goal;
wakeUp(goal);
}
return goal;
}
template<typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<StorePath, std::weak_ptr<G>> & goalMap)
std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
{
std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
auto goal = goal_weak.lock(); // FIXME
if (!goal) {
goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, repair, ca);
goal_weak = goal;
wakeUp(goal);
}
return goal;
}
template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{
/* !!! inefficient */
for (auto i = goalMap.begin();
@ -109,8 +122,10 @@ void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else if (auto subGoal = std::dynamic_pointer_cast<SubstitutionGoal>(goal))
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals);
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
else
assert(false);
if (topGoals.find(goal) != topGoals.end()) {
@ -217,7 +232,7 @@ void Worker::run(const Goals & _topGoals)
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
topPaths.push_back({goal->drvPath, goal->wantedOutputs});
} else if (auto goal = dynamic_cast<SubstitutionGoal *>(i.get())) {
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back({goal->storePath});
}
}
@ -471,7 +486,10 @@ void Worker::markContentsGood(const StorePath & path)
}
GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal) {
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
return subGoal;
}
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
return subGoal;
}

17
src/libstore/build/worker.hh
View file

@ -4,6 +4,7 @@
#include "lock.hh"
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
#include <future>
#include <thread>
@ -12,18 +13,20 @@ namespace nix {
/* Forward definition. */
struct DerivationGoal;
struct SubstitutionGoal;
struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal;
/* Workaround for not being able to declare a something like
class SubstitutionGoal : public Goal;
class PathSubstitutionGoal : public Goal;
even when Goal is a complete type.
This is still a static cast. The purpose of exporting it is to define it in
a place where `SubstitutionGoal` is concrete, and use it in a place where it
a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
is opaque. */
GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
@ -72,7 +75,8 @@ private:
/* Maps used to prevent multiple instantiations of a goal for the
same derivation / path. */
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<SubstitutionGoal>> substitutionGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
/* Goals waiting for busy paths to be unlocked. */
WeakGoals waitingForAnyGoal;
@ -146,7 +150,8 @@ public:
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */
std::shared_ptr<SubstitutionGoal> makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);

1
src/libstore/ca-specific-schema.sql
View file

@ -6,6 +6,7 @@ create table if not exists Realisations (
drvPath text not null,
outputName text not null, -- symbolic output id, usually "out"
outputPath integer not null,
signatures text, -- space-separated list
primary key (drvPath, outputName),
foreign key (outputPath) references ValidPaths(id) on delete cascade
);

5
src/libstore/daemon.cc
View file

@ -575,7 +575,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto res = store->buildDerivation(drvPath, drv, buildMode);
logger->stopWork();
to << res.status << res.errorMsg;
if (GET_PROTOCOL_MINOR(clientVersion) >= 0xc) {
if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
}
if (GET_PROTOCOL_MINOR(clientVersion) >= 28) {
worker_proto::write(*store, to, res.builtOutputs);
}
break;

2
src/libstore/derivations.hh
View file

@ -52,7 +52,7 @@ struct DerivationOutput
DerivationOutputCAFloating,
DerivationOutputDeferred
> output;
std::optional<HashType> hashAlgoOpt(const Store & store) const;
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
interface provided by BasicDerivation::outputsAndOptPaths */

2
src/libstore/globals.cc
View file

@ -81,7 +81,7 @@ void loadConfFile()
/* We only want to send overrides to the daemon, i.e. stuff from
~/.nix/nix.conf or the command line. */
globalConfig.resetOverriden();
globalConfig.resetOverridden();
auto files = settings.nixUserConfFiles;
for (auto file = files.rbegin(); file != files.rend(); file++) {

5
src/libstore/globals.hh
View file

@ -206,7 +206,10 @@ public:
Setting<std::string> builders{
this, "@" + nixConfDir + "/machines", "builders",
"A semicolon-separated list of build machines, in the format of `nix.machines`."};
R"(
A semicolon-separated list of build machines.
For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md)
)"};
Setting<bool> buildersUseSubstitutes{
this, false, "builders-use-substitutes",

2
src/libstore/legacy-ssh-store.cc
View file

@ -15,6 +15,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig
using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
@ -59,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
, master(
host,
sshKey,
sshPublicHostKey,
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
compress,

5
src/libstore/local-binary-cache-store.cc
View file

@ -2,6 +2,8 @@
#include "globals.hh"
#include "nar-info-disk-cache.hh"
#include <atomic>
namespace nix {
struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
@ -50,7 +52,8 @@ protected:
const std::string & mimeType) override
{
auto path2 = binaryCacheDir + "/" + path;
Path tmp = path2 + ".tmp." + std::to_string(getpid());
static std::atomic<int> counter{0};
Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter);
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);

41
src/libstore/local-store.cc
View file

@ -310,13 +310,13 @@ LocalStore::LocalStore(const Params & params)
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
state->stmts->RegisterRealisedOutput.create(state->db,
R"(
insert or replace into Realisations (drvPath, outputName, outputPath)
values (?, ?, (select id from ValidPaths where path = ?))
insert or replace into Realisations (drvPath, outputName, outputPath, signatures)
values (?, ?, (select id from ValidPaths where path = ?), ?)
;
)");
state->stmts->QueryRealisedOutput.create(state->db,
R"(
select Output.path from Realisations
select Output.path, Realisations.signatures from Realisations
inner join ValidPaths as Output on Output.id = Realisations.outputPath
where drvPath = ? and outputName = ?
;
@ -652,6 +652,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
}
}
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
{
settings.requireExperimentalFeature("ca-derivations");
if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info))
registerDrvOutput(info);
else
throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string());
}
void LocalStore::registerDrvOutput(const Realisation & info)
{
@ -662,6 +670,7 @@ void LocalStore::registerDrvOutput(const Realisation & info)
(info.id.strHash())
(info.id.outputName)
(printStorePath(info.outPath))
(concatStringsSep(" ", info.signatures))
.exec();
});
}
@ -1108,15 +1117,20 @@ const PublicKeys & LocalStore::getPublicKeys()
return *state->publicKeys;
}
bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info)
bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info)
{
return requireSigs && !info.checkSignatures(*this, getPublicKeys());
}
bool LocalStore::realisationIsUntrusted(const Realisation & realisation)
{
return requireSigs && !realisation.checkSignatures(getPublicKeys());
}
void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs)
{
if (checkSigs && pathInfoIsTrusted(info))
if (checkSigs && pathInfoIsUntrusted(info))
throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path));
addTempRoot(info.path);
@ -1630,6 +1644,18 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si
}
void LocalStore::signRealisation(Realisation & realisation)
{
// FIXME: keep secret keys in memory.
auto secretKeyFiles = settings.secretKeyFiles;
for (auto & secretKeyFile : secretKeyFiles.get()) {
SecretKey secretKey(readFile(secretKeyFile));
realisation.sign(secretKey);
}
}
void LocalStore::signPathInfo(ValidPathInfo & info)
{
// FIXME: keep secret keys in memory.
@ -1667,8 +1693,9 @@ std::optional<const Realisation> LocalStore::queryRealisation(
if (!use.next())
return std::nullopt;
auto outputPath = parseStorePath(use.getStr(0));
return Ret{
Realisation{.id = id, .outPath = outputPath}};
auto signatures = tokenizeString<StringSet>(use.getStr(1));
return Ret{Realisation{
.id = id, .outPath = outputPath, .signatures = signatures}};
});
}
} // namespace nix

9
src/libstore/local-store.hh
View file

@ -136,7 +136,8 @@ public:
void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
bool pathInfoIsTrusted(const ValidPathInfo &) override;
bool pathInfoIsUntrusted(const ValidPathInfo &) override;
bool realisationIsUntrusted(const Realisation & ) override;
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override;
@ -202,6 +203,7 @@ public:
/* Register the store path 'output' as the output named 'outputName' of
derivation 'deriver'. */
void registerDrvOutput(const Realisation & info) override;
void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
@ -272,16 +274,19 @@ private:
bool isValidPath_(State & state, const StorePath & path);
void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers);
/* Add signatures to a ValidPathInfo using the secret keys
/* Add signatures to a ValidPathInfo or Realisation using the secret keys
specified by the ‘secret-key-files’ option. */
void signPathInfo(ValidPathInfo & info);
void signRealisation(Realisation &);
Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override;
friend struct LocalDerivationGoal;
friend struct PathSubstitutionGoal;
friend struct SubstitutionGoal;
friend struct DerivationGoal;
};

2
src/libstore/local.mk
View file

@ -28,7 +28,7 @@ ifeq ($(OS), SunOS)
endif
ifeq ($(HAVE_SECCOMP), 1)
libstore_LDFLAGS += -lseccomp
libstore_LDFLAGS += $(LIBSECCOMP_LIBS)
endif
libstore_CXXFLAGS += \

6
src/libstore/machines.cc
View file

@ -54,9 +54,15 @@ ref<Store> Machine::openStore() const {
if (hasPrefix(storeUri, "ssh://")) {
storeParams["max-connections"] = "1";
storeParams["log-fd"] = "4";
}
if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) {
if (sshKey != "")
storeParams["ssh-key"] = sshKey;
if (sshPublicHostKey != "")
storeParams["base64-ssh-public-host-key"] = sshPublicHostKey;
}
{
auto & fs = storeParams["system-features"];
auto append = [&](auto feats) {

35
src/libstore/misc.cc
View file

@ -22,54 +22,52 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
Sync<State> state_(State{0, paths_, 0});
std::function<void(const Path &)> enqueue;
std::function<void(const StorePath &)> enqueue;
std::condition_variable done;
enqueue = [&](const Path & path) -> void {
enqueue = [&](const StorePath & path) -> void {
{
auto state(state_.lock());
if (state->exc) return;
if (!state->paths.insert(parseStorePath(path)).second) return;
if (!state->paths.insert(path).second) return;
state->pending++;
}
queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future<ref<const ValidPathInfo>> fut) {
queryPathInfo(path, {[&](std::future<ref<const ValidPathInfo>> fut) {
// FIXME: calls to isValidPath() should be async
try {
auto info = fut.get();
auto path = parseStorePath(pathS);
if (flipDirection) {
StorePathSet referrers;
queryReferrers(path, referrers);
for (auto & ref : referrers)
if (ref != path)
enqueue(printStorePath(ref));
enqueue(ref);
if (includeOutputs)
for (auto & i : queryValidDerivers(path))
enqueue(printStorePath(i));
enqueue(i);
if (includeDerivers && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
enqueue(printStorePath(i));
enqueue(i);
} else {
for (auto & ref : info->references)
enqueue(printStorePath(ref));
enqueue(ref);
if (includeOutputs && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i)) enqueue(printStorePath(i));
if (isValidPath(i)) enqueue(i);
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
enqueue(printStorePath(*info->deriver));
enqueue(*info->deriver);
}
@ -89,7 +87,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
};
for (auto & startPath : startPaths)
enqueue(printStorePath(startPath));
enqueue(startPath);
{
auto state(state_.lock());
@ -159,13 +157,10 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
};
auto checkOutput = [&](
const Path & drvPathS, ref<Derivation> drv, const Path & outPathS, ref<Sync<DrvState>> drvState_)
const StorePath & drvPath, ref<Derivation> drv, const StorePath & outPath, ref<Sync<DrvState>> drvState_)
{
if (drvState_->lock()->done) return;
auto drvPath = parseStorePath(drvPathS);
auto outPath = parseStorePath(outPathS);
SubstitutablePathInfos infos;
querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
@ -202,7 +197,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
return;
}
PathSet invalid;
StorePathSet invalid;
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
@ -212,7 +207,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
break;
}
if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
invalid.insert(printStorePath(*pathOpt));
invalid.insert(*pathOpt);
}
if (knownOutputPaths && invalid.empty()) return;
@ -222,7 +217,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState));
} else
mustBuildDrv(path.path, *drv);

46
src/libstore/realisation.cc
View file

@ -25,27 +25,69 @@ nlohmann::json Realisation::toJSON() const {
return nlohmann::json{
{"id", id.to_string()},
{"outPath", outPath.to_string()},
{"signatures", signatures},
};
}
Realisation Realisation::fromJSON(
const nlohmann::json& json,
const std::string& whence) {
auto getField = [&](std::string fieldName) -> std::string {
auto getOptionalField = [&](std::string fieldName) -> std::optional<std::string> {
auto fieldIterator = json.find(fieldName);
if (fieldIterator == json.end())
return std::nullopt;
return *fieldIterator;
};
auto getField = [&](std::string fieldName) -> std::string {
if (auto field = getOptionalField(fieldName))
return *field;
else
throw Error(
"Drv output info file '%1%' is corrupt, missing field %2%",
whence, fieldName);
return *fieldIterator;
};
StringSet signatures;
if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
return Realisation{
.id = DrvOutput::parse(getField("id")),
.outPath = StorePath(getField("outPath")),
.signatures = signatures,
};
}
std::string Realisation::fingerprint() const
{
auto serialized = toJSON();
serialized.erase("signatures");
return serialized.dump();
}
void Realisation::sign(const SecretKey & secretKey)
{
signatures.insert(secretKey.signDetached(fingerprint()));
}
bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
{
return verifyDetached(fingerprint(), sig, publicKeys);
}
size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const
{
// FIXME: Maybe we should return `maxSigs` if the realisation corresponds to
// an input-addressed one − because in that case the drv is enough to check
// it − but we can't know that here.
size_t good = 0;
for (auto & sig : signatures)
if (checkSignature(publicKeys, sig))
good++;
return good;
}
StorePath RealisedPath::path() const {
return std::visit([](auto && arg) { return arg.getPath(); }, raw);
}

8
src/libstore/realisation.hh
View file

@ -6,6 +6,7 @@
#include "path.hh"
#include <nlohmann/json_fwd.hpp>
#include "comparator.hh"
#include "crypto.hh"
namespace nix {
@ -28,9 +29,16 @@ struct Realisation {
DrvOutput id;
StorePath outPath;
StringSet signatures;
nlohmann::json toJSON() const;
static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
std::string fingerprint() const;
void sign(const SecretKey &);
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
size_t checkSignatures(const PublicKeys & publicKeys) const;
StorePath getPath() const { return outPath; }
GENERATE_CMP(Realisation, me->id, me->outPath);

20
src/libstore/remote-store.cc
View file

@ -62,9 +62,15 @@ void write(const Store & store, Sink & out, const Realisation & realisation)
{ out << realisation.toJSON().dump(); }
DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
{ return DrvOutput::parse(readString(from)); }
{
return DrvOutput::parse(readString(from));
}
void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
{ out << drvOutput.to_string(); }
{
out << drvOutput.to_string();
}
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
{
@ -678,10 +684,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
conn->to << buildMode;
conn.processStderr();
BuildResult res;
unsigned int status;
conn->from >> status >> res.errorMsg;
res.status = (BuildResult::Status) status;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0xc) {
res.status = (BuildResult::Status) readInt(conn->from);
conn->from >> res.errorMsg;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
res.builtOutputs = builtOutputs;
}

2
src/libstore/serve-protocol.hh
View file

@ -5,7 +5,7 @@ namespace nix {
#define SERVE_MAGIC_1 0x390c9deb
#define SERVE_MAGIC_2 0x5452eecb
#define SERVE_PROTOCOL_VERSION 0x206
#define SERVE_PROTOCOL_VERSION (2 << 8 | 6)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)

2
src/libstore/ssh-store.cc
View file

@ -13,6 +13,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig
using RemoteStoreConfig::RemoteStoreConfig;
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
@ -34,6 +35,7 @@ public:
, master(
host,
sshKey,
sshPublicHostKey,
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
compress)

16
src/libstore/ssh.cc
View file

@ -2,24 +2,37 @@
namespace nix {
SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD)
SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD)
: host(host)
, fakeSSH(host == "localhost")
, keyFile(keyFile)
, sshPublicHostKey(sshPublicHostKey)
, useMaster(useMaster && !fakeSSH)
, compress(compress)
, logFD(logFD)
{
if (host == "" || hasPrefix(host, "-"))
throw Error("invalid SSH host name '%s'", host);
auto state(state_.lock());
state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
}
void SSHMaster::addCommonSSHOpts(Strings & args)
{
auto state(state_.lock());
for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS").value_or("")))
args.push_back(i);
if (!keyFile.empty())
args.insert(args.end(), {"-i", keyFile});
if (!sshPublicHostKey.empty()) {
Path fileName = (Path) *state->tmpDir + "/host-key";
auto p = host.rfind("@");
string thost = p != string::npos ? string(host, p + 1) : host;
writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n");
args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName});
}
if (compress)
args.push_back("-C");
}
@ -87,7 +100,6 @@ Path SSHMaster::startMaster()
if (state->sshMaster != -1) return state->socketPath;
state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
state->socketPath = (Path) *state->tmpDir + "/ssh.sock";

3
src/libstore/ssh.hh
View file

@ -12,6 +12,7 @@ private:
const std::string host;
bool fakeSSH;
const std::string keyFile;
const std::string sshPublicHostKey;
const bool useMaster;
const bool compress;
const int logFD;
@ -29,7 +30,7 @@ private:
public:
SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1);
SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1);
struct Connection
{

2
src/libstore/store-api.cc
View file

@ -815,7 +815,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
try {
for (auto & realisation : realisations) {
dstStore->registerDrvOutput(realisation);
dstStore->registerDrvOutput(realisation, checkSigs);
}
} catch (MissingExperimentalFeature & e) {
// Don't fail if the remote doesn't support CA derivations is it might

9
src/libstore/store-api.hh
View file

@ -380,7 +380,12 @@ public:
we don't really want to add the dependencies listed in a nar info we
don't trust anyyways.
*/
virtual bool pathInfoIsTrusted(const ValidPathInfo &)
virtual bool pathInfoIsUntrusted(const ValidPathInfo &)
{
return true;
}
virtual bool realisationIsUntrusted(const Realisation & )
{
return true;
}
@ -476,6 +481,8 @@ public:
*/
virtual void registerDrvOutput(const Realisation & output)
{ unsupported("registerDrvOutput"); }
virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs)
{ return registerDrvOutput(output); }
/* Write a NAR dump of a store path. */
virtual void narFromPath(const StorePath & path, Sink & sink) = 0;

2
src/libstore/worker-protocol.hh
View file

@ -9,7 +9,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x11c
#define PROTOCOL_VERSION (1 << 8 | 29)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)