mirror of
https://github.com/NixOS/nix.git
synced 2025-12-14 21:11:04 +01:00
Merge remote-tracking branch 'upstream/master' into auto-uid-allocation
This commit is contained in:
John Ericson
commit
e023c985d5
383 changed files with 16916 additions and 9809 deletions
File diff suppressed because it is too large
Load diff
|
|
@ -2,7 +2,8 @@
|
|||
|
||||
#include "parsed-derivations.hh"
|
||||
#include "lock.hh"
|
||||
#include "local-store.hh"
|
||||
#include "store-api.hh"
|
||||
#include "pathlocks.hh"
|
||||
#include "goal.hh"
|
||||
|
||||
namespace nix {
|
||||
|
|
@ -37,18 +38,21 @@ struct InitialOutputStatus {
|
|||
|
||||
struct InitialOutput {
|
||||
bool wanted;
|
||||
Hash outputHash;
|
||||
std::optional<InitialOutputStatus> known;
|
||||
};
|
||||
|
||||
class DerivationGoal : public Goal
|
||||
struct DerivationGoal : public Goal
|
||||
{
|
||||
private:
|
||||
/* Whether to use an on-disk .drv file. */
|
||||
bool useDerivation;
|
||||
|
||||
/* The path of the derivation. */
|
||||
StorePath drvPath;
|
||||
|
||||
/* The path of the corresponding resolved derivation */
|
||||
std::optional<BasicDerivation> resolvedDrv;
|
||||
|
||||
/* The specific outputs that we need to build. Empty means all of
|
||||
them. */
|
||||
StringSet wantedOutputs;
|
||||
|
|
@ -61,7 +65,7 @@ private:
|
|||
bool retrySubstitution;
|
||||
|
||||
/* The derivation stored at drvPath. */
|
||||
std::unique_ptr<BasicDerivation> drv;
|
||||
std::unique_ptr<Derivation> drv;
|
||||
|
||||
std::unique_ptr<ParsedDerivation> parsedDrv;
|
||||
|
||||
|
|
@ -76,18 +80,6 @@ private:
|
|||
|
||||
std::map<std::string, InitialOutput> initialOutputs;
|
||||
|
||||
/* User selected for running the builder. */
|
||||
std::unique_ptr<UserLock> buildUser;
|
||||
|
||||
/* The process ID of the builder. */
|
||||
Pid pid;
|
||||
|
||||
/* The temporary directory. */
|
||||
Path tmpDir;
|
||||
|
||||
/* The path of the temporary directory in the sandbox. */
|
||||
Path tmpDirInSandbox;
|
||||
|
||||
/* File descriptor for the log file. */
|
||||
AutoCloseFD fdLogFile;
|
||||
std::shared_ptr<BufferedSink> logFileSink, logSink;
|
||||
|
|
@ -103,86 +95,15 @@ private:
|
|||
|
||||
std::string currentHookLine;
|
||||
|
||||
/* Pipe for the builder's standard output/error. */
|
||||
Pipe builderOut;
|
||||
|
||||
/* Pipe for synchronising updates to the builder namespaces. */
|
||||
Pipe userNamespaceSync;
|
||||
|
||||
/* The mount namespace of the builder, used to add additional
|
||||
paths to the sandbox as a result of recursive Nix calls. */
|
||||
AutoCloseFD sandboxMountNamespace;
|
||||
|
||||
/* On Linux, whether we're doing the build in its own user
|
||||
namespace. */
|
||||
bool usingUserNamespace = true;
|
||||
|
||||
/* The build hook. */
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
/* Whether we're currently doing a chroot build. */
|
||||
bool useChroot = false;
|
||||
|
||||
Path chrootRootDir;
|
||||
|
||||
/* Whether to give the build more than 1 UID. */
|
||||
bool useUidRange = false;
|
||||
|
||||
/* Whether to make the 'systemd' cgroup controller available to
|
||||
the build. */
|
||||
bool useSystemdCgroup = false;
|
||||
|
||||
/* RAII object to delete the chroot directory. */
|
||||
std::shared_ptr<AutoDelete> autoDelChroot;
|
||||
|
||||
/* The sort of derivation we are building. */
|
||||
DerivationType derivationType;
|
||||
|
||||
/* Whether to run the build in a private network namespace. */
|
||||
bool privateNetwork = false;
|
||||
|
||||
typedef void (DerivationGoal::*GoalState)();
|
||||
GoalState state;
|
||||
|
||||
/* Stuff we need to pass to initChild(). */
|
||||
struct ChrootPath {
|
||||
Path source;
|
||||
bool optional;
|
||||
ChrootPath(Path source = "", bool optional = false)
|
||||
: source(source), optional(optional)
|
||||
{ }
|
||||
};
|
||||
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
|
||||
DirsInChroot dirsInChroot;
|
||||
|
||||
typedef map<string, string> Environment;
|
||||
Environment env;
|
||||
|
||||
#if __APPLE__
|
||||
typedef string SandboxProfile;
|
||||
SandboxProfile additionalSandboxProfile;
|
||||
#endif
|
||||
|
||||
/* Hash rewriting. */
|
||||
StringMap inputRewrites, outputRewrites;
|
||||
typedef map<StorePath, StorePath> RedirectedOutputs;
|
||||
RedirectedOutputs redirectedOutputs;
|
||||
|
||||
/* The outputs paths used during the build.
|
||||
|
||||
- Input-addressed derivations or fixed content-addressed outputs are
|
||||
sometimes built when some of their outputs already exist, and can not
|
||||
be hidden via sandboxing. We use temporary locations instead and
|
||||
rewrite after the build. Otherwise the regular predetermined paths are
|
||||
put here.
|
||||
|
||||
- Floating content-addressed derivations do not know their final build
|
||||
output paths until the outputs are hashed, so random locations are
|
||||
used, and then renamed. The randomness helps guard against hidden
|
||||
self-references.
|
||||
*/
|
||||
OutputPathMap scratchOutputs;
|
||||
|
||||
/* The final output paths of the build.
|
||||
|
||||
- For input-addressed derivations, always the precomputed paths
|
||||
|
|
@ -195,11 +116,6 @@ private:
|
|||
|
||||
BuildMode buildMode;
|
||||
|
||||
/* If we're repairing without a chroot, there may be outputs that
|
||||
are valid but corrupt. So we redirect these outputs to
|
||||
temporary paths. */
|
||||
StorePathSet redirectedBadOutputs;
|
||||
|
||||
BuildResult result;
|
||||
|
||||
/* The current round, if we're building multiple times. */
|
||||
|
|
@ -207,17 +123,6 @@ private:
|
|||
|
||||
size_t nrRounds;
|
||||
|
||||
/* Path registration info from the previous round, if we're
|
||||
building multiple times. Since this contains the hash, it
|
||||
allows us to compare whether two rounds produced the same
|
||||
result. */
|
||||
std::map<Path, ValidPathInfo> prevInfos;
|
||||
|
||||
uid_t sandboxUid() { return usingUserNamespace ? (useUidRange ? 0 : 1000) : buildUser->getUID(); }
|
||||
gid_t sandboxGid() { return usingUserNamespace ? (useUidRange ? 0 : 100) : buildUser->getGID(); }
|
||||
|
||||
const static Path homeDir;
|
||||
|
||||
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
|
||||
|
||||
std::unique_ptr<Activity> act;
|
||||
|
|
@ -230,40 +135,13 @@ private:
|
|||
/* The remote machine on which we're building. */
|
||||
std::string machineName;
|
||||
|
||||
/* The recursive Nix daemon socket. */
|
||||
AutoCloseFD daemonSocket;
|
||||
|
||||
/* The daemon main thread. */
|
||||
std::thread daemonThread;
|
||||
|
||||
/* The daemon worker threads. */
|
||||
std::vector<std::thread> daemonWorkerThreads;
|
||||
|
||||
/* Paths that were added via recursive Nix calls. */
|
||||
StorePathSet addedPaths;
|
||||
|
||||
/* Recursive Nix calls are only allowed to build or realize paths
|
||||
in the original input closure or added via a recursive Nix call
|
||||
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
|
||||
/nix/store/<bla> is some arbitrary path in a binary cache). */
|
||||
bool isAllowed(const StorePath & path)
|
||||
{
|
||||
return inputPaths.count(path) || addedPaths.count(path);
|
||||
}
|
||||
|
||||
friend struct RestrictedStore;
|
||||
|
||||
public:
|
||||
DerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const StringSet & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
~DerivationGoal();
|
||||
|
||||
/* Whether we need to perform hash rewriting if there are valid output paths. */
|
||||
bool needsHashRewrite();
|
||||
virtual ~DerivationGoal();
|
||||
|
||||
void timedOut(Error && ex) override;
|
||||
|
||||
|
|
@ -271,17 +149,11 @@ public:
|
|||
|
||||
void work() override;
|
||||
|
||||
StorePath getDrvPath()
|
||||
{
|
||||
return drvPath;
|
||||
}
|
||||
|
||||
/* Add wanted outputs to an already existing derivation goal. */
|
||||
void addWantedOutputs(const StringSet & outputs);
|
||||
|
||||
BuildResult getResult() { return result; }
|
||||
|
||||
private:
|
||||
/* The states. */
|
||||
void getDerivation();
|
||||
void loadDerivation();
|
||||
|
|
@ -291,7 +163,7 @@ private:
|
|||
void closureRepaired();
|
||||
void inputsRealised();
|
||||
void tryToBuild();
|
||||
void tryLocalBuild();
|
||||
virtual void tryLocalBuild();
|
||||
void buildDone();
|
||||
|
||||
void resolvedFinished();
|
||||
|
|
@ -299,51 +171,33 @@ private:
|
|||
/* Is the build hook willing to perform the build? */
|
||||
HookReply tryBuildHook();
|
||||
|
||||
/* Start building a derivation. */
|
||||
void startBuilder();
|
||||
|
||||
/* Fill in the environment for the builder. */
|
||||
void initEnv();
|
||||
|
||||
/* Setup tmp dir location. */
|
||||
void initTmpDir();
|
||||
|
||||
/* Write a JSON file containing the derivation attributes. */
|
||||
void writeStructuredAttrs();
|
||||
|
||||
void startDaemon();
|
||||
|
||||
void stopDaemon();
|
||||
|
||||
/* Add 'path' to the set of paths that may be referenced by the
|
||||
outputs, and make it appear in the sandbox. */
|
||||
void addDependency(const StorePath & path);
|
||||
|
||||
/* Make a file owned by the builder. */
|
||||
void chownToBuilder(const Path & path);
|
||||
|
||||
/* Run the builder's process. */
|
||||
void runChild();
|
||||
|
||||
friend int childEntry(void *);
|
||||
virtual int getChildStatus();
|
||||
|
||||
/* Check that the derivation outputs all exist and register them
|
||||
as valid. */
|
||||
void registerOutputs();
|
||||
|
||||
/* Check that an output meets the requirements specified by the
|
||||
'outputChecks' attribute (or the legacy
|
||||
'{allowed,disallowed}{References,Requisites}' attributes). */
|
||||
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
|
||||
virtual void registerOutputs();
|
||||
|
||||
/* Open a log file and a pipe to it. */
|
||||
Path openLogFile();
|
||||
|
||||
/* Sign the newly built realisation if the store allows it */
|
||||
virtual void signRealisation(Realisation&) {}
|
||||
|
||||
/* Close the log file. */
|
||||
void closeLogFile();
|
||||
|
||||
/* Delete the temporary directory, if we have one. */
|
||||
void deleteTmpDir(bool force);
|
||||
/* Close the read side of the logger pipe. */
|
||||
virtual void closeReadPipes();
|
||||
|
||||
/* Cleanup hooks for buildDone() */
|
||||
virtual void cleanupHookFinally();
|
||||
virtual void cleanupPreChildKill();
|
||||
virtual void cleanupPostChildKill();
|
||||
virtual bool cleanupDecideWhetherDiskFull();
|
||||
virtual void cleanupPostOutputsRegisteredModeCheck();
|
||||
virtual void cleanupPostOutputsRegisteredModeNonCheck();
|
||||
|
||||
virtual bool isReadDesc(int fd);
|
||||
|
||||
/* Callback used by the worker to write to the log. */
|
||||
void handleChildOutput(int fd, const string & data) override;
|
||||
|
|
@ -360,17 +214,7 @@ private:
|
|||
void checkPathValidity();
|
||||
|
||||
/* Forcibly kill the child process, if any. */
|
||||
void killChild();
|
||||
|
||||
/* Create alternative path calculated from but distinct from the
|
||||
input, so we can avoid overwriting outputs (or other store paths)
|
||||
that already exist. */
|
||||
StorePath makeFallbackPath(const StorePath & path);
|
||||
/* Make a path to another based on the output name along with the
|
||||
derivation hash. */
|
||||
/* FIXME add option to randomize, so we can audit whether our
|
||||
rewrites caught everything */
|
||||
StorePath makeFallbackPath(std::string_view outputName);
|
||||
virtual void killChild();
|
||||
|
||||
void repairClosure();
|
||||
|
||||
|
|
@ -383,4 +227,6 @@ private:
|
|||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
};
|
||||
|
||||
MakeError(NotDeterministic, BuildError);
|
||||
|
||||
}
|
||||
|
|
|
|||
95
src/libstore/build/drv-output-substitution-goal.cc
Normal file
95
src/libstore/build/drv-output-substitution-goal.cc
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
#include "drv-output-substitution-goal.hh"
|
||||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
: Goal(worker)
|
||||
, id(id)
|
||||
{
|
||||
state = &DrvOutputSubstitutionGoal::init;
|
||||
name = fmt("substitution of '%s'", id.to_string());
|
||||
trace("created");
|
||||
}
|
||||
|
||||
|
||||
void DrvOutputSubstitutionGoal::init()
|
||||
{
|
||||
trace("init");
|
||||
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
|
||||
tryNext();
|
||||
}
|
||||
|
||||
void DrvOutputSubstitutionGoal::tryNext()
|
||||
{
|
||||
trace("Trying next substituter");
|
||||
|
||||
if (subs.size() == 0) {
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
with it. */
|
||||
debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
|
||||
|
||||
/* Hack: don't indicate failure if there were no substituters.
|
||||
In that case the calling derivation should just do a
|
||||
build. */
|
||||
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
|
||||
|
||||
if (substituterFailed) {
|
||||
worker.failedSubstitutions++;
|
||||
worker.updateProgress();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
auto sub = subs.front();
|
||||
subs.pop_front();
|
||||
|
||||
// FIXME: Make async
|
||||
outputInfo = sub->queryRealisation(id);
|
||||
if (!outputInfo) {
|
||||
tryNext();
|
||||
return;
|
||||
}
|
||||
|
||||
addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
|
||||
|
||||
if (waitees.empty()) outPathValid();
|
||||
else state = &DrvOutputSubstitutionGoal::outPathValid;
|
||||
}
|
||||
|
||||
void DrvOutputSubstitutionGoal::outPathValid()
|
||||
{
|
||||
assert(outputInfo);
|
||||
trace("Output path substituted");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
|
||||
return;
|
||||
}
|
||||
|
||||
worker.store.registerDrvOutput(*outputInfo);
|
||||
finished();
|
||||
}
|
||||
|
||||
void DrvOutputSubstitutionGoal::finished()
|
||||
{
|
||||
trace("finished");
|
||||
amDone(ecSuccess);
|
||||
}
|
||||
|
||||
string DrvOutputSubstitutionGoal::key()
|
||||
{
|
||||
/* "a$" ensures substitution goals happen before derivation
|
||||
goals. */
|
||||
return "a$" + std::string(id.to_string());
|
||||
}
|
||||
|
||||
void DrvOutputSubstitutionGoal::work()
|
||||
{
|
||||
(this->*state)();
|
||||
}
|
||||
|
||||
}
|
||||
50
src/libstore/build/drv-output-substitution-goal.hh
Normal file
50
src/libstore/build/drv-output-substitution-goal.hh
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
#pragma once
|
||||
|
||||
#include "store-api.hh"
|
||||
#include "goal.hh"
|
||||
#include "realisation.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class Worker;
|
||||
|
||||
// Substitution of a derivation output.
|
||||
// This is done in three steps:
|
||||
// 1. Fetch the output info from a substituter
|
||||
// 2. Substitute the corresponding output path
|
||||
// 3. Register the output info
|
||||
class DrvOutputSubstitutionGoal : public Goal {
|
||||
private:
|
||||
// The drv output we're trying to substitue
|
||||
DrvOutput id;
|
||||
|
||||
// The realisation corresponding to the given output id.
|
||||
// Will be filled once we can get it.
|
||||
std::optional<Realisation> outputInfo;
|
||||
|
||||
/* The remaining substituters. */
|
||||
std::list<ref<Store>> subs;
|
||||
|
||||
/* Whether a substituter failed. */
|
||||
bool substituterFailed = false;
|
||||
|
||||
public:
|
||||
DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
|
||||
typedef void (DrvOutputSubstitutionGoal::*GoalState)();
|
||||
GoalState state;
|
||||
|
||||
void init();
|
||||
void tryNext();
|
||||
void outPathValid();
|
||||
void finished();
|
||||
|
||||
void timedOut(Error && ex) override { abort(); };
|
||||
|
||||
string key() override;
|
||||
|
||||
void work() override;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -2,34 +2,24 @@
|
|||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
#include "derivation-goal.hh"
|
||||
#include "local-store.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
|
||||
{
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
uint64_t downloadSize, narSize;
|
||||
store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
|
||||
if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
|
||||
throw Error(
|
||||
"%d derivations need to be built, but neither local builds ('--max-jobs') "
|
||||
"nor remote builds ('--builders') are enabled", willBuild.size());
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
|
||||
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode)
|
||||
{
|
||||
Worker worker(*this);
|
||||
|
||||
primeCache(*this, drvPaths);
|
||||
|
||||
Goals goals;
|
||||
for (auto & path : drvPaths) {
|
||||
if (path.path.isDerivation())
|
||||
goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
|
||||
else
|
||||
goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
|
||||
for (auto & br : reqs) {
|
||||
std::visit(overloaded {
|
||||
[&](DerivedPath::Built bfd) {
|
||||
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
|
||||
},
|
||||
[&](DerivedPath::Opaque bo) {
|
||||
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
|
||||
},
|
||||
}, br.raw());
|
||||
}
|
||||
|
||||
worker.run(goals);
|
||||
|
|
@ -44,9 +34,8 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths,
|
|||
ex = i->ex;
|
||||
}
|
||||
if (i->exitCode != Goal::ecSuccess) {
|
||||
DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
|
||||
if (i2) failed.insert(i2->getDrvPath());
|
||||
else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
|
||||
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
|
||||
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -59,7 +48,7 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths,
|
|||
}
|
||||
}
|
||||
|
||||
BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
BuildMode buildMode)
|
||||
{
|
||||
Worker worker(*this);
|
||||
|
|
@ -74,20 +63,38 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe
|
|||
result.status = BuildResult::MiscFailure;
|
||||
result.errorMsg = e.msg();
|
||||
}
|
||||
// XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
|
||||
// extended to return the full realisation for each output
|
||||
auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
|
||||
auto outputHashes = staticOutputHashes(*this, drv);
|
||||
for (auto & [outputName, staticOutput] : staticDrvOutputs) {
|
||||
auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
|
||||
if (staticOutput.second)
|
||||
result.builtOutputs.insert_or_assign(
|
||||
outputId,
|
||||
Realisation{ outputId, *staticOutput.second}
|
||||
);
|
||||
if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
|
||||
auto realisation = this->queryRealisation(outputId);
|
||||
if (realisation)
|
||||
result.builtOutputs.insert_or_assign(
|
||||
outputId,
|
||||
*realisation
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::ensurePath(const StorePath & path)
|
||||
void Store::ensurePath(const StorePath & path)
|
||||
{
|
||||
/* If the path is already valid, we're done. */
|
||||
if (isValidPath(path)) return;
|
||||
|
||||
primeCache(*this, {{path}});
|
||||
|
||||
Worker worker(*this);
|
||||
GoalPtr goal = worker.makeSubstitutionGoal(path);
|
||||
GoalPtr goal = worker.makePathSubstitutionGoal(path);
|
||||
Goals goals = {goal};
|
||||
|
||||
worker.run(goals);
|
||||
|
|
@ -105,7 +112,7 @@ void LocalStore::ensurePath(const StorePath & path)
|
|||
void LocalStore::repairPath(const StorePath & path)
|
||||
{
|
||||
Worker worker(*this);
|
||||
GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
|
||||
GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
|
||||
Goals goals = {goal};
|
||||
|
||||
worker.run(goals);
|
||||
|
|
@ -78,6 +78,8 @@ void Goal::amDone(ExitCode result, std::optional<Error> ex)
|
|||
}
|
||||
waiters.clear();
|
||||
worker.removeGoal(shared_from_this());
|
||||
|
||||
cleanup();
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ namespace nix {
|
|||
|
||||
/* Forward definition. */
|
||||
struct Goal;
|
||||
struct Worker;
|
||||
class Worker;
|
||||
|
||||
/* A pointer to a goal. */
|
||||
typedef std::shared_ptr<Goal> GoalPtr;
|
||||
|
|
@ -46,7 +46,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
|
|||
unsigned int nrNoSubstituters;
|
||||
|
||||
/* Number of substitution goals we are/were waiting for that
|
||||
failed because othey had unsubstitutable references. */
|
||||
failed because they had unsubstitutable references. */
|
||||
unsigned int nrIncompleteClosure;
|
||||
|
||||
/* Name of this goal for debugging purposes. */
|
||||
|
|
@ -100,6 +100,8 @@ struct Goal : public std::enable_shared_from_this<Goal>
|
|||
virtual string key() = 0;
|
||||
|
||||
void amDone(ExitCode result, std::optional<Error> ex = {});
|
||||
|
||||
virtual void cleanup() { }
|
||||
};
|
||||
|
||||
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
|
||||
|
|
|
|||
2906
src/libstore/build/local-derivation-goal.cc
Normal file
2906
src/libstore/build/local-derivation-goal.cc
Normal file
File diff suppressed because it is too large
Load diff
209
src/libstore/build/local-derivation-goal.hh
Normal file
209
src/libstore/build/local-derivation-goal.hh
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
#pragma once
|
||||
|
||||
#include "derivation-goal.hh"
|
||||
#include "local-store.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct LocalDerivationGoal : public DerivationGoal
|
||||
{
|
||||
LocalStore & getLocalStore();
|
||||
|
||||
/* User selected for running the builder. */
|
||||
std::unique_ptr<UserLock> buildUser;
|
||||
|
||||
/* The process ID of the builder. */
|
||||
Pid pid;
|
||||
|
||||
/* The temporary directory. */
|
||||
Path tmpDir;
|
||||
|
||||
/* The path of the temporary directory in the sandbox. */
|
||||
Path tmpDirInSandbox;
|
||||
|
||||
/* Pipe for the builder's standard output/error. */
|
||||
Pipe builderOut;
|
||||
|
||||
/* Pipe for synchronising updates to the builder namespaces. */
|
||||
Pipe userNamespaceSync;
|
||||
|
||||
/* The mount namespace of the builder, used to add additional
|
||||
paths to the sandbox as a result of recursive Nix calls. */
|
||||
AutoCloseFD sandboxMountNamespace;
|
||||
|
||||
/* On Linux, whether we're doing the build in its own user
|
||||
namespace. */
|
||||
bool usingUserNamespace = true;
|
||||
|
||||
/* Whether we're currently doing a chroot build. */
|
||||
bool useChroot = false;
|
||||
|
||||
Path chrootRootDir;
|
||||
|
||||
/* Whether to give the build more than 1 UID. */
|
||||
bool useUidRange = false;
|
||||
|
||||
/* Whether to make the 'systemd' cgroup controller available to
|
||||
the build. */
|
||||
bool useSystemdCgroup = false;
|
||||
|
||||
/* RAII object to delete the chroot directory. */
|
||||
std::shared_ptr<AutoDelete> autoDelChroot;
|
||||
|
||||
/* Whether to run the build in a private network namespace. */
|
||||
bool privateNetwork = false;
|
||||
|
||||
/* Stuff we need to pass to initChild(). */
|
||||
struct ChrootPath {
|
||||
Path source;
|
||||
bool optional;
|
||||
ChrootPath(Path source = "", bool optional = false)
|
||||
: source(source), optional(optional)
|
||||
{ }
|
||||
};
|
||||
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
|
||||
DirsInChroot dirsInChroot;
|
||||
|
||||
typedef map<string, string> Environment;
|
||||
Environment env;
|
||||
|
||||
#if __APPLE__
|
||||
typedef string SandboxProfile;
|
||||
SandboxProfile additionalSandboxProfile;
|
||||
#endif
|
||||
|
||||
/* Hash rewriting. */
|
||||
StringMap inputRewrites, outputRewrites;
|
||||
typedef map<StorePath, StorePath> RedirectedOutputs;
|
||||
RedirectedOutputs redirectedOutputs;
|
||||
|
||||
/* The outputs paths used during the build.
|
||||
|
||||
- Input-addressed derivations or fixed content-addressed outputs are
|
||||
sometimes built when some of their outputs already exist, and can not
|
||||
be hidden via sandboxing. We use temporary locations instead and
|
||||
rewrite after the build. Otherwise the regular predetermined paths are
|
||||
put here.
|
||||
|
||||
- Floating content-addressed derivations do not know their final build
|
||||
output paths until the outputs are hashed, so random locations are
|
||||
used, and then renamed. The randomness helps guard against hidden
|
||||
self-references.
|
||||
*/
|
||||
OutputPathMap scratchOutputs;
|
||||
|
||||
/* Path registration info from the previous round, if we're
|
||||
building multiple times. Since this contains the hash, it
|
||||
allows us to compare whether two rounds produced the same
|
||||
result. */
|
||||
std::map<Path, ValidPathInfo> prevInfos;
|
||||
|
||||
uid_t sandboxUid() { return usingUserNamespace ? (useUidRange ? 0 : 1000) : buildUser->getUID(); }
|
||||
gid_t sandboxGid() { return usingUserNamespace ? (useUidRange ? 0 : 100) : buildUser->getGID(); }
|
||||
|
||||
const static Path homeDir;
|
||||
|
||||
/* The recursive Nix daemon socket. */
|
||||
AutoCloseFD daemonSocket;
|
||||
|
||||
/* The daemon main thread. */
|
||||
std::thread daemonThread;
|
||||
|
||||
/* The daemon worker threads. */
|
||||
std::vector<std::thread> daemonWorkerThreads;
|
||||
|
||||
/* Paths that were added via recursive Nix calls. */
|
||||
StorePathSet addedPaths;
|
||||
|
||||
/* Recursive Nix calls are only allowed to build or realize paths
|
||||
in the original input closure or added via a recursive Nix call
|
||||
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
|
||||
/nix/store/<bla> is some arbitrary path in a binary cache). */
|
||||
bool isAllowed(const StorePath & path)
|
||||
{
|
||||
return inputPaths.count(path) || addedPaths.count(path);
|
||||
}
|
||||
bool isAllowed(const DerivedPath & req);
|
||||
|
||||
friend struct RestrictedStore;
|
||||
|
||||
using DerivationGoal::DerivationGoal;
|
||||
|
||||
virtual ~LocalDerivationGoal() override;
|
||||
|
||||
/* Whether we need to perform hash rewriting if there are valid output paths. */
|
||||
bool needsHashRewrite();
|
||||
|
||||
/* The additional states. */
|
||||
void tryLocalBuild() override;
|
||||
|
||||
/* Start building a derivation. */
|
||||
void startBuilder();
|
||||
|
||||
/* Fill in the environment for the builder. */
|
||||
void initEnv();
|
||||
|
||||
/* Setup tmp dir location. */
|
||||
void initTmpDir();
|
||||
|
||||
/* Write a JSON file containing the derivation attributes. */
|
||||
void writeStructuredAttrs();
|
||||
|
||||
void startDaemon();
|
||||
|
||||
void stopDaemon();
|
||||
|
||||
/* Add 'path' to the set of paths that may be referenced by the
|
||||
outputs, and make it appear in the sandbox. */
|
||||
void addDependency(const StorePath & path);
|
||||
|
||||
/* Make a file owned by the builder. */
|
||||
void chownToBuilder(const Path & path);
|
||||
|
||||
int getChildStatus() override;
|
||||
|
||||
/* Run the builder's process. */
|
||||
void runChild();
|
||||
|
||||
/* Check that the derivation outputs all exist and register them
|
||||
as valid. */
|
||||
void registerOutputs() override;
|
||||
|
||||
void signRealisation(Realisation &) override;
|
||||
|
||||
/* Check that an output meets the requirements specified by the
|
||||
'outputChecks' attribute (or the legacy
|
||||
'{allowed,disallowed}{References,Requisites}' attributes). */
|
||||
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
|
||||
|
||||
/* Close the read side of the logger pipe. */
|
||||
void closeReadPipes() override;
|
||||
|
||||
/* Cleanup hooks for buildDone() */
|
||||
void cleanupHookFinally() override;
|
||||
void cleanupPreChildKill() override;
|
||||
void cleanupPostChildKill() override;
|
||||
bool cleanupDecideWhetherDiskFull() override;
|
||||
void cleanupPostOutputsRegisteredModeCheck() override;
|
||||
void cleanupPostOutputsRegisteredModeNonCheck() override;
|
||||
|
||||
bool isReadDesc(int fd) override;
|
||||
|
||||
/* Delete the temporary directory, if we have one. */
|
||||
void deleteTmpDir(bool force);
|
||||
|
||||
/* Forcibly kill the child process, if any. */
|
||||
void killChild() override;
|
||||
|
||||
/* Create alternative path calculated from but distinct from the
|
||||
input, so we can avoid overwriting outputs (or other store paths)
|
||||
that already exist. */
|
||||
StorePath makeFallbackPath(const StorePath & path);
|
||||
/* Make a path to another based on the output name along with the
|
||||
derivation hash. */
|
||||
/* FIXME add option to randomize, so we can audit whether our
|
||||
rewrites caught everything */
|
||||
StorePath makeFallbackPath(std::string_view outputName);
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -5,40 +5,32 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
: Goal(worker)
|
||||
, storePath(storePath)
|
||||
, repair(repair)
|
||||
, ca(ca)
|
||||
{
|
||||
state = &SubstitutionGoal::init;
|
||||
state = &PathSubstitutionGoal::init;
|
||||
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
|
||||
trace("created");
|
||||
maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
|
||||
}
|
||||
|
||||
|
||||
SubstitutionGoal::~SubstitutionGoal()
|
||||
PathSubstitutionGoal::~PathSubstitutionGoal()
|
||||
{
|
||||
try {
|
||||
if (thr.joinable()) {
|
||||
// FIXME: signal worker thread to quit.
|
||||
thr.join();
|
||||
worker.childTerminated(this);
|
||||
}
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
cleanup();
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::work()
|
||||
void PathSubstitutionGoal::work()
|
||||
{
|
||||
(this->*state)();
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::init()
|
||||
void PathSubstitutionGoal::init()
|
||||
{
|
||||
trace("init");
|
||||
|
||||
|
|
@ -59,10 +51,12 @@ void SubstitutionGoal::init()
|
|||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::tryNext()
|
||||
void PathSubstitutionGoal::tryNext()
|
||||
{
|
||||
trace("trying next substituter");
|
||||
|
||||
cleanup();
|
||||
|
||||
if (subs.size() == 0) {
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
with it. */
|
||||
|
|
@ -142,15 +136,10 @@ void SubstitutionGoal::tryNext()
|
|||
/* Bail out early if this substituter lacks a valid
|
||||
signature. LocalStore::addToStore() also checks for this, but
|
||||
only after we've downloaded the path. */
|
||||
if (worker.store.requireSigs
|
||||
&& !sub->isTrusted
|
||||
&& !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
|
||||
if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
|
||||
{
|
||||
logWarning({
|
||||
.name = "Invalid path signature",
|
||||
.hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
|
||||
sub->getUri(), worker.store.printStorePath(storePath))
|
||||
});
|
||||
warn("substituter '%s' does not have a valid signature for path '%s'",
|
||||
sub->getUri(), worker.store.printStorePath(storePath));
|
||||
tryNext();
|
||||
return;
|
||||
}
|
||||
|
|
@ -159,16 +148,16 @@ void SubstitutionGoal::tryNext()
|
|||
paths referenced by this one. */
|
||||
for (auto & i : info->references)
|
||||
if (i != storePath) /* ignore self-references */
|
||||
addWaitee(worker.makeSubstitutionGoal(i));
|
||||
addWaitee(worker.makePathSubstitutionGoal(i));
|
||||
|
||||
if (waitees.empty()) /* to prevent hang (no wake-up event) */
|
||||
referencesValid();
|
||||
else
|
||||
state = &SubstitutionGoal::referencesValid;
|
||||
state = &PathSubstitutionGoal::referencesValid;
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::referencesValid()
|
||||
void PathSubstitutionGoal::referencesValid()
|
||||
{
|
||||
trace("all references realised");
|
||||
|
||||
|
|
@ -182,12 +171,12 @@ void SubstitutionGoal::referencesValid()
|
|||
if (i != storePath) /* ignore self-references */
|
||||
assert(worker.store.isValidPath(i));
|
||||
|
||||
state = &SubstitutionGoal::tryToRun;
|
||||
state = &PathSubstitutionGoal::tryToRun;
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::tryToRun()
|
||||
void PathSubstitutionGoal::tryToRun()
|
||||
{
|
||||
trace("trying to run");
|
||||
|
||||
|
|
@ -210,7 +199,7 @@ void SubstitutionGoal::tryToRun()
|
|||
thr = std::thread([this]() {
|
||||
try {
|
||||
/* Wake up the worker loop when we're done. */
|
||||
Finally updateStats([this]() { outPipe.writeSide = -1; });
|
||||
Finally updateStats([this]() { outPipe.writeSide.close(); });
|
||||
|
||||
Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
|
||||
PushActivity pact(act.id);
|
||||
|
|
@ -226,11 +215,11 @@ void SubstitutionGoal::tryToRun()
|
|||
|
||||
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
|
||||
|
||||
state = &SubstitutionGoal::finished;
|
||||
state = &PathSubstitutionGoal::finished;
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::finished()
|
||||
void PathSubstitutionGoal::finished()
|
||||
{
|
||||
trace("substitute finished");
|
||||
|
||||
|
|
@ -254,7 +243,7 @@ void SubstitutionGoal::finished()
|
|||
}
|
||||
|
||||
/* Try the next substitute. */
|
||||
state = &SubstitutionGoal::tryNext;
|
||||
state = &PathSubstitutionGoal::tryNext;
|
||||
worker.wakeUp(shared_from_this());
|
||||
return;
|
||||
}
|
||||
|
|
@ -283,14 +272,31 @@ void SubstitutionGoal::finished()
|
|||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::handleChildOutput(int fd, const string & data)
|
||||
void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void SubstitutionGoal::handleEOF(int fd)
|
||||
void PathSubstitutionGoal::handleEOF(int fd)
|
||||
{
|
||||
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
|
||||
void PathSubstitutionGoal::cleanup()
|
||||
{
|
||||
try {
|
||||
if (thr.joinable()) {
|
||||
// FIXME: signal worker thread to quit.
|
||||
thr.join();
|
||||
worker.childTerminated(this);
|
||||
}
|
||||
|
||||
outPipe.close();
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,16 +8,13 @@ namespace nix {
|
|||
|
||||
class Worker;
|
||||
|
||||
class SubstitutionGoal : public Goal
|
||||
struct PathSubstitutionGoal : public Goal
|
||||
{
|
||||
friend class Worker;
|
||||
|
||||
private:
|
||||
/* The store path that should be realised through a substitute. */
|
||||
StorePath storePath;
|
||||
|
||||
/* The path the substituter refers to the path as. This will be
|
||||
* different when the stores have different names. */
|
||||
different when the stores have different names. */
|
||||
std::optional<StorePath> subPath;
|
||||
|
||||
/* The remaining substituters. */
|
||||
|
|
@ -50,15 +47,15 @@ private:
|
|||
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
|
||||
maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
|
||||
|
||||
typedef void (SubstitutionGoal::*GoalState)();
|
||||
typedef void (PathSubstitutionGoal::*GoalState)();
|
||||
GoalState state;
|
||||
|
||||
/* Content address for recomputing store path */
|
||||
std::optional<ContentAddress> ca;
|
||||
|
||||
public:
|
||||
SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
~SubstitutionGoal();
|
||||
PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
~PathSubstitutionGoal();
|
||||
|
||||
void timedOut(Error && ex) override { abort(); };
|
||||
|
||||
|
|
@ -83,7 +80,7 @@ public:
|
|||
void handleChildOutput(int fd, const string & data) override;
|
||||
void handleEOF(int fd) override;
|
||||
|
||||
StorePath getStorePath() { return storePath; }
|
||||
void cleanup() override;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
#include "machines.hh"
|
||||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
#include "derivation-goal.hh"
|
||||
#include "drv-output-substitution-goal.hh"
|
||||
#include "local-derivation-goal.hh"
|
||||
#include "hook-instance.hh"
|
||||
|
||||
#include <poll.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
Worker::Worker(LocalStore & store)
|
||||
Worker::Worker(Store & store)
|
||||
: act(*logger, actRealise)
|
||||
, actDerivations(*logger, actBuilds)
|
||||
, actSubstitutions(*logger, actCopyPaths)
|
||||
|
|
@ -43,16 +44,13 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
|
|||
const StringSet & wantedOutputs,
|
||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
|
||||
{
|
||||
WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
|
||||
GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
|
||||
std::shared_ptr<DerivationGoal> goal;
|
||||
if (!abstract_goal) {
|
||||
std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
|
||||
std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
|
||||
if (!goal) {
|
||||
goal = mkDrvGoal();
|
||||
abstract_goal_weak = goal;
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
} else {
|
||||
goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
|
||||
assert(goal);
|
||||
goal->addWantedOutputs(wantedOutputs);
|
||||
}
|
||||
return goal;
|
||||
|
|
@ -62,8 +60,10 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
|
|||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode)
|
||||
{
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
|
||||
return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
|
||||
return !dynamic_cast<LocalStore *>(&store)
|
||||
? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
|
||||
: std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -71,32 +71,46 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
|
|||
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
|
||||
const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
|
||||
{
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
|
||||
return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
|
||||
return !dynamic_cast<LocalStore *>(&store)
|
||||
? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
|
||||
: std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
{
|
||||
WeakGoalPtr & goal_weak = substitutionGoals[path];
|
||||
GoalPtr goal = goal_weak.lock(); // FIXME
|
||||
std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
|
||||
auto goal = goal_weak.lock(); // FIXME
|
||||
if (!goal) {
|
||||
goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
|
||||
goal = std::make_shared<PathSubstitutionGoal>(path, *this, repair, ca);
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
}
|
||||
return goal;
|
||||
}
|
||||
|
||||
std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
{
|
||||
std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
|
||||
auto goal = goal_weak.lock(); // FIXME
|
||||
if (!goal) {
|
||||
goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, repair, ca);
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
}
|
||||
return goal;
|
||||
}
|
||||
|
||||
static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
|
||||
template<typename K, typename G>
|
||||
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
|
||||
{
|
||||
/* !!! inefficient */
|
||||
for (WeakGoalMap::iterator i = goalMap.begin();
|
||||
for (auto i = goalMap.begin();
|
||||
i != goalMap.end(); )
|
||||
if (i->second.lock() == goal) {
|
||||
WeakGoalMap::iterator j = i; ++j;
|
||||
auto j = i; ++j;
|
||||
goalMap.erase(i);
|
||||
i = j;
|
||||
}
|
||||
|
|
@ -106,8 +120,15 @@ static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
|
|||
|
||||
void Worker::removeGoal(GoalPtr goal)
|
||||
{
|
||||
nix::removeGoal(goal, derivationGoals);
|
||||
nix::removeGoal(goal, substitutionGoals);
|
||||
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
|
||||
nix::removeGoal(drvGoal, derivationGoals);
|
||||
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
|
||||
nix::removeGoal(subGoal, substitutionGoals);
|
||||
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
|
||||
nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
|
||||
else
|
||||
assert(false);
|
||||
|
||||
if (topGoals.find(goal) != topGoals.end()) {
|
||||
topGoals.erase(goal);
|
||||
/* If a top-level goal failed, then kill all other goals
|
||||
|
|
@ -206,7 +227,21 @@ void Worker::waitForAWhile(GoalPtr goal)
|
|||
|
||||
void Worker::run(const Goals & _topGoals)
|
||||
{
|
||||
for (auto & i : _topGoals) topGoals.insert(i);
|
||||
std::vector<nix::DerivedPath> topPaths;
|
||||
|
||||
for (auto & i : _topGoals) {
|
||||
topGoals.insert(i);
|
||||
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
|
||||
topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs});
|
||||
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
|
||||
topPaths.push_back(DerivedPath::Opaque{goal->storePath});
|
||||
}
|
||||
}
|
||||
|
||||
/* Call queryMissing() efficiently query substitutes. */
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
uint64_t downloadSize, narSize;
|
||||
store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
|
||||
debug("entered goal loop");
|
||||
|
||||
|
|
@ -214,7 +249,9 @@ void Worker::run(const Goals & _topGoals)
|
|||
|
||||
checkInterrupt();
|
||||
|
||||
store.autoGC(false);
|
||||
// TODO GC interface?
|
||||
if (auto localStore = dynamic_cast<LocalStore *>(&store))
|
||||
localStore->autoGC(false);
|
||||
|
||||
/* Call every wake goal (in the ordering established by
|
||||
CompareGoalPtrs). */
|
||||
|
|
@ -439,10 +476,7 @@ bool Worker::pathContentsGood(const StorePath & path)
|
|||
}
|
||||
pathContentsGoodCache.insert_or_assign(path, res);
|
||||
if (!res)
|
||||
logError({
|
||||
.name = "Corrupted path",
|
||||
.hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
|
||||
});
|
||||
printError("path '%s' is corrupted or missing!", store.printStorePath(path));
|
||||
return res;
|
||||
}
|
||||
|
||||
|
|
@ -452,4 +486,12 @@ void Worker::markContentsGood(const StorePath & path)
|
|||
pathContentsGoodCache.insert_or_assign(path, true);
|
||||
}
|
||||
|
||||
|
||||
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
|
||||
return subGoal;
|
||||
}
|
||||
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
|
||||
return subGoal;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,13 +2,31 @@
|
|||
|
||||
#include "types.hh"
|
||||
#include "lock.hh"
|
||||
#include "local-store.hh"
|
||||
#include "store-api.hh"
|
||||
#include "goal.hh"
|
||||
#include "realisation.hh"
|
||||
|
||||
#include <future>
|
||||
#include <thread>
|
||||
|
||||
namespace nix {
|
||||
|
||||
/* Forward definition. */
|
||||
class DerivationGoal;
|
||||
struct DerivationGoal;
|
||||
struct PathSubstitutionGoal;
|
||||
class DrvOutputSubstitutionGoal;
|
||||
|
||||
/* Workaround for not being able to declare a something like
|
||||
|
||||
class PathSubstitutionGoal : public Goal;
|
||||
|
||||
even when Goal is a complete type.
|
||||
|
||||
This is still a static cast. The purpose of exporting it is to define it in
|
||||
a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
|
||||
is opaque. */
|
||||
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
|
||||
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
|
||||
|
||||
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
|
||||
|
||||
|
|
@ -56,8 +74,9 @@ private:
|
|||
|
||||
/* Maps used to prevent multiple instantiations of a goal for the
|
||||
same derivation / path. */
|
||||
WeakGoalMap derivationGoals;
|
||||
WeakGoalMap substitutionGoals;
|
||||
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
|
||||
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
|
||||
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
|
||||
|
||||
/* Goals waiting for busy paths to be unlocked. */
|
||||
WeakGoals waitingForAnyGoal;
|
||||
|
|
@ -90,7 +109,7 @@ public:
|
|||
/* Set if at least one derivation is not deterministic in check mode. */
|
||||
bool checkMismatch;
|
||||
|
||||
LocalStore & store;
|
||||
Store & store;
|
||||
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
|
|
@ -112,7 +131,7 @@ public:
|
|||
it answers with "decline-permanently", we don't try again. */
|
||||
bool tryBuildHook = true;
|
||||
|
||||
Worker(LocalStore & store);
|
||||
Worker(Store & store);
|
||||
~Worker();
|
||||
|
||||
/* Make a goal (with caching). */
|
||||
|
|
@ -131,7 +150,8 @@ public:
|
|||
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
|
||||
/* substitution goal */
|
||||
GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
|
||||
/* Remove a dead goal. */
|
||||
void removeGoal(GoalPtr goal);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue