mirror of
https://github.com/NixOS/nix.git
synced 2025-12-11 11:31:03 +01:00
Split {,local-}derivation-goal.{cc,hh}
This separates the scheduling logic (including simple hook pathway) from the local-store needing code. This should be the final split for now. I'm reasonably happy with how it's turning out, even before I'm done moving code into `local-derivation-goal`. Benefits: 1. This will help "witness" that the hook case is indeed a lot simpler, and also compensate for the increased complexity that comes from content-addressed derivation outputs. 2. It also moves us ever so slightly towards a world where we could use off-the-shelf storage or sandboxing, since `local-derivation-goal` would be gutted in those cases, but `derivation-goal` should remain nearly the same. The new `#if 0` in the new files will be deleted in the following commit. I keep it here so if it turns out more stuff can be moved over, it's easy to do so in a way that preserves ordering --- and thus prevents conflicts. N.B. ```sh git diff HEAD^^ --color-moved --find-copies-harder --patience --stat ``` makes nicer output.
This commit is contained in:
John Ericson
parent
05cc5a8587
commit
68f4c728ec
7 changed files with 329 additions and 3045 deletions
|
|
@ -2,7 +2,8 @@
|
|||
|
||||
#include "parsed-derivations.hh"
|
||||
#include "lock.hh"
|
||||
#include "local-store.hh"
|
||||
#include "store-api.hh"
|
||||
#include "pathlocks.hh"
|
||||
#include "goal.hh"
|
||||
|
||||
namespace nix {
|
||||
|
|
@ -79,18 +80,6 @@ struct DerivationGoal : public Goal
|
|||
|
||||
std::map<std::string, InitialOutput> initialOutputs;
|
||||
|
||||
/* User selected for running the builder. */
|
||||
std::unique_ptr<UserLock> buildUser;
|
||||
|
||||
/* The process ID of the builder. */
|
||||
Pid pid;
|
||||
|
||||
/* The temporary directory. */
|
||||
Path tmpDir;
|
||||
|
||||
/* The path of the temporary directory in the sandbox. */
|
||||
Path tmpDirInSandbox;
|
||||
|
||||
/* File descriptor for the log file. */
|
||||
AutoCloseFD fdLogFile;
|
||||
std::shared_ptr<BufferedSink> logFileSink, logSink;
|
||||
|
|
@ -106,79 +95,15 @@ struct DerivationGoal : public Goal
|
|||
|
||||
std::string currentHookLine;
|
||||
|
||||
/* Pipe for the builder's standard output/error. */
|
||||
Pipe builderOut;
|
||||
|
||||
/* Pipe for synchronising updates to the builder namespaces. */
|
||||
Pipe userNamespaceSync;
|
||||
|
||||
/* The mount namespace of the builder, used to add additional
|
||||
paths to the sandbox as a result of recursive Nix calls. */
|
||||
AutoCloseFD sandboxMountNamespace;
|
||||
|
||||
/* On Linux, whether we're doing the build in its own user
|
||||
namespace. */
|
||||
bool usingUserNamespace = true;
|
||||
|
||||
/* The build hook. */
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
/* Whether we're currently doing a chroot build. */
|
||||
bool useChroot = false;
|
||||
|
||||
Path chrootRootDir;
|
||||
|
||||
/* RAII object to delete the chroot directory. */
|
||||
std::shared_ptr<AutoDelete> autoDelChroot;
|
||||
|
||||
/* The sort of derivation we are building. */
|
||||
DerivationType derivationType;
|
||||
|
||||
/* Whether to run the build in a private network namespace. */
|
||||
bool privateNetwork = false;
|
||||
|
||||
typedef void (DerivationGoal::*GoalState)();
|
||||
GoalState state;
|
||||
|
||||
/* Stuff we need to pass to initChild(). */
|
||||
struct ChrootPath {
|
||||
Path source;
|
||||
bool optional;
|
||||
ChrootPath(Path source = "", bool optional = false)
|
||||
: source(source), optional(optional)
|
||||
{ }
|
||||
};
|
||||
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
|
||||
DirsInChroot dirsInChroot;
|
||||
|
||||
typedef map<string, string> Environment;
|
||||
Environment env;
|
||||
|
||||
#if __APPLE__
|
||||
typedef string SandboxProfile;
|
||||
SandboxProfile additionalSandboxProfile;
|
||||
#endif
|
||||
|
||||
/* Hash rewriting. */
|
||||
StringMap inputRewrites, outputRewrites;
|
||||
typedef map<StorePath, StorePath> RedirectedOutputs;
|
||||
RedirectedOutputs redirectedOutputs;
|
||||
|
||||
/* The outputs paths used during the build.
|
||||
|
||||
- Input-addressed derivations or fixed content-addressed outputs are
|
||||
sometimes built when some of their outputs already exist, and can not
|
||||
be hidden via sandboxing. We use temporary locations instead and
|
||||
rewrite after the build. Otherwise the regular predetermined paths are
|
||||
put here.
|
||||
|
||||
- Floating content-addressed derivations do not know their final build
|
||||
output paths until the outputs are hashed, so random locations are
|
||||
used, and then renamed. The randomness helps guard against hidden
|
||||
self-references.
|
||||
*/
|
||||
OutputPathMap scratchOutputs;
|
||||
|
||||
/* The final output paths of the build.
|
||||
|
||||
- For input-addressed derivations, always the precomputed paths
|
||||
|
|
@ -191,11 +116,6 @@ struct DerivationGoal : public Goal
|
|||
|
||||
BuildMode buildMode;
|
||||
|
||||
/* If we're repairing without a chroot, there may be outputs that
|
||||
are valid but corrupt. So we redirect these outputs to
|
||||
temporary paths. */
|
||||
StorePathSet redirectedBadOutputs;
|
||||
|
||||
BuildResult result;
|
||||
|
||||
/* The current round, if we're building multiple times. */
|
||||
|
|
@ -203,17 +123,6 @@ struct DerivationGoal : public Goal
|
|||
|
||||
size_t nrRounds;
|
||||
|
||||
/* Path registration info from the previous round, if we're
|
||||
building multiple times. Since this contains the hash, it
|
||||
allows us to compare whether two rounds produced the same
|
||||
result. */
|
||||
std::map<Path, ValidPathInfo> prevInfos;
|
||||
|
||||
uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
|
||||
gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); }
|
||||
|
||||
const static Path homeDir;
|
||||
|
||||
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
|
||||
|
||||
std::unique_ptr<Activity> act;
|
||||
|
|
@ -226,39 +135,13 @@ struct DerivationGoal : public Goal
|
|||
/* The remote machine on which we're building. */
|
||||
std::string machineName;
|
||||
|
||||
/* The recursive Nix daemon socket. */
|
||||
AutoCloseFD daemonSocket;
|
||||
|
||||
/* The daemon main thread. */
|
||||
std::thread daemonThread;
|
||||
|
||||
/* The daemon worker threads. */
|
||||
std::vector<std::thread> daemonWorkerThreads;
|
||||
|
||||
/* Paths that were added via recursive Nix calls. */
|
||||
StorePathSet addedPaths;
|
||||
|
||||
/* Recursive Nix calls are only allowed to build or realize paths
|
||||
in the original input closure or added via a recursive Nix call
|
||||
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
|
||||
/nix/store/<bla> is some arbitrary path in a binary cache). */
|
||||
bool isAllowed(const StorePath & path)
|
||||
{
|
||||
return inputPaths.count(path) || addedPaths.count(path);
|
||||
}
|
||||
|
||||
friend struct RestrictedStore;
|
||||
|
||||
DerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const StringSet & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
~DerivationGoal();
|
||||
|
||||
/* Whether we need to perform hash rewriting if there are valid output paths. */
|
||||
bool needsHashRewrite();
|
||||
virtual ~DerivationGoal();
|
||||
|
||||
void timedOut(Error && ex) override;
|
||||
|
||||
|
|
@ -280,7 +163,7 @@ struct DerivationGoal : public Goal
|
|||
void closureRepaired();
|
||||
void inputsRealised();
|
||||
void tryToBuild();
|
||||
void tryLocalBuild();
|
||||
virtual void tryLocalBuild();
|
||||
void buildDone();
|
||||
|
||||
void resolvedFinished();
|
||||
|
|
@ -288,40 +171,11 @@ struct DerivationGoal : public Goal
|
|||
/* Is the build hook willing to perform the build? */
|
||||
HookReply tryBuildHook();
|
||||
|
||||
/* Start building a derivation. */
|
||||
void startBuilder();
|
||||
|
||||
/* Fill in the environment for the builder. */
|
||||
void initEnv();
|
||||
|
||||
/* Setup tmp dir location. */
|
||||
void initTmpDir();
|
||||
|
||||
/* Write a JSON file containing the derivation attributes. */
|
||||
void writeStructuredAttrs();
|
||||
|
||||
void startDaemon();
|
||||
|
||||
void stopDaemon();
|
||||
|
||||
/* Add 'path' to the set of paths that may be referenced by the
|
||||
outputs, and make it appear in the sandbox. */
|
||||
void addDependency(const StorePath & path);
|
||||
|
||||
/* Make a file owned by the builder. */
|
||||
void chownToBuilder(const Path & path);
|
||||
|
||||
/* Run the builder's process. */
|
||||
void runChild();
|
||||
virtual int getChildStatus();
|
||||
|
||||
/* Check that the derivation outputs all exist and register them
|
||||
as valid. */
|
||||
void registerOutputs();
|
||||
|
||||
/* Check that an output meets the requirements specified by the
|
||||
'outputChecks' attribute (or the legacy
|
||||
'{allowed,disallowed}{References,Requisites}' attributes). */
|
||||
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
|
||||
virtual void registerOutputs();
|
||||
|
||||
/* Open a log file and a pipe to it. */
|
||||
Path openLogFile();
|
||||
|
|
@ -329,8 +183,18 @@ struct DerivationGoal : public Goal
|
|||
/* Close the log file. */
|
||||
void closeLogFile();
|
||||
|
||||
/* Delete the temporary directory, if we have one. */
|
||||
void deleteTmpDir(bool force);
|
||||
/* Close the read side of the logger pipe. */
|
||||
virtual void closeReadPipes();
|
||||
|
||||
/* Cleanup hooks for buildDone() */
|
||||
virtual void cleanupHookFinally();
|
||||
virtual void cleanupPreChildKill();
|
||||
virtual void cleanupPostChildKill();
|
||||
virtual bool cleanupDecideWhetherDiskFull();
|
||||
virtual void cleanupPostOutputsRegisteredModeCheck();
|
||||
virtual void cleanupPostOutputsRegisteredModeNonCheck();
|
||||
|
||||
virtual bool isReadDesc(int fd);
|
||||
|
||||
/* Callback used by the worker to write to the log. */
|
||||
void handleChildOutput(int fd, const string & data) override;
|
||||
|
|
@ -347,17 +211,7 @@ struct DerivationGoal : public Goal
|
|||
void checkPathValidity();
|
||||
|
||||
/* Forcibly kill the child process, if any. */
|
||||
void killChild();
|
||||
|
||||
/* Create alternative path calculated from but distinct from the
|
||||
input, so we can avoid overwriting outputs (or other store paths)
|
||||
that already exist. */
|
||||
StorePath makeFallbackPath(const StorePath & path);
|
||||
/* Make a path to another based on the output name along with the
|
||||
derivation hash. */
|
||||
/* FIXME add option to randomize, so we can audit whether our
|
||||
rewrites caught everything */
|
||||
StorePath makeFallbackPath(std::string_view outputName);
|
||||
virtual void killChild();
|
||||
|
||||
void repairClosure();
|
||||
|
||||
|
|
@ -370,4 +224,6 @@ struct DerivationGoal : public Goal
|
|||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
};
|
||||
|
||||
MakeError(NotDeterministic, BuildError);
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue