nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-03 07:31:00 +01:00
Code Activity

Merge remote-tracking branch 'upstream/master' into ca-drv-exotic

Browse source
This commit is contained in:
John Ericson 2023-04-17 10:16:57 -04:00
commit e12efa3654
246 changed files with 5067 additions and 1911 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/libstore/build/derivation-goal.cc
View file

@ -911,7 +911,11 @@ void DerivationGoal::buildDone()
msg += line;
msg += "\n";
}
msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.",
auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand)
? "nix log"
: "nix-store -l";
msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.",
nixLogCommand,
worker.store.printStorePath(drvPath));
}

175
src/libstore/build/derivation-goal.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "parsed-derivations.hh"
#include "lock.hh"
@ -15,8 +16,10 @@ struct HookInstance;
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
/* Unless we are repairing, we don't both to test validity and just assume it,
so the choices are `Absent` or `Valid`. */
/**
* Unless we are repairing, we don't both to test validity and just assume it,
* so the choices are `Absent` or `Valid`.
*/
enum struct PathStatus {
Corrupt,
Absent,
@ -26,11 +29,15 @@ enum struct PathStatus {
struct InitialOutputStatus {
StorePath path;
PathStatus status;
/* Valid in the store, and additionally non-corrupt if we are repairing */
/**
* Valid in the store, and additionally non-corrupt if we are repairing
*/
bool isValid() const {
return status == PathStatus::Valid;
}
/* Merely present, allowed to be corrupt */
/**
* Merely present, allowed to be corrupt
*/
bool isPresent() const {
return status == PathStatus::Corrupt
|| status == PathStatus::Valid;
@ -45,59 +52,87 @@ struct InitialOutput {
struct DerivationGoal : public Goal
{
/* Whether to use an on-disk .drv file. */
/**
* Whether to use an on-disk .drv file.
*/
bool useDerivation;
/* The path of the derivation. */
/** The path of the derivation. */
StorePath drvPath;
/* The goal for the corresponding resolved derivation */
/**
* The goal for the corresponding resolved derivation
*/
std::shared_ptr<DerivationGoal> resolvedDrvGoal;
/* The specific outputs that we need to build. Empty means all of
them. */
/**
* The specific outputs that we need to build. Empty means all of
* them.
*/
OutputsSpec wantedOutputs;
/* Mapping from input derivations + output names to actual store
paths. This is filled in by waiteeDone() as each dependency
finishes, before inputsRealised() is reached, */
/**
* Mapping from input derivations + output names to actual store
* paths. This is filled in by waiteeDone() as each dependency
* finishes, before inputsRealised() is reached.
*/
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/* Whether additional wanted outputs have been added. */
/**
* Whether additional wanted outputs have been added.
*/
bool needRestart = false;
/* Whether to retry substituting the outputs after building the
inputs. This is done in case of an incomplete closure. */
/**
* Whether to retry substituting the outputs after building the
* inputs. This is done in case of an incomplete closure.
*/
bool retrySubstitution = false;
/* Whether we've retried substitution, in which case we won't try
again. */
/**
* Whether we've retried substitution, in which case we won't try
* again.
*/
bool retriedSubstitution = false;
/* The derivation stored at drvPath. */
/**
* The derivation stored at drvPath.
*/
std::unique_ptr<Derivation> drv;
std::unique_ptr<ParsedDerivation> parsedDrv;
/* The remainder is state held during the build. */
/**
* The remainder is state held during the build.
*/
/* Locks on (fixed) output paths. */
/**
* Locks on (fixed) output paths.
*/
PathLocks outputLocks;
/* All input paths (that is, the union of FS closures of the
immediate input paths). */
/**
* All input paths (that is, the union of FS closures of the
* immediate input paths).
*/
StorePathSet inputPaths;
std::map<std::string, InitialOutput> initialOutputs;
/* File descriptor for the log file. */
/**
* File descriptor for the log file.
*/
AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink;
/* Number of bytes received from the builder's stdout/stderr. */
/**
* Number of bytes received from the builder's stdout/stderr.
*/
unsigned long logSize;
/* The most recent log lines. */
/**
* The most recent log lines.
*/
std::list<std::string> logTail;
std::string currentLogLine;
@ -105,10 +140,14 @@ struct DerivationGoal : public Goal
std::string currentHookLine;
/* The build hook. */
/**
* The build hook.
*/
std::unique_ptr<HookInstance> hook;
/* The sort of derivation we are building. */
/**
* The sort of derivation we are building.
*/
DerivationType derivationType;
typedef void (DerivationGoal::*GoalState)();
@ -120,12 +159,16 @@ struct DerivationGoal : public Goal
std::unique_ptr<Activity> act;
/* Activity that denotes waiting for a lock. */
/**
* Activity that denotes waiting for a lock.
*/
std::unique_ptr<Activity> actLock;
std::map<ActivityId, Activity> builderActivities;
/* The remote machine on which we're building. */
/**
* The remote machine on which we're building.
*/
std::string machineName;
DerivationGoal(const StorePath & drvPath,
@ -142,10 +185,14 @@ struct DerivationGoal : public Goal
void work() override;
/* Add wanted outputs to an already existing derivation goal. */
/**
* Add wanted outputs to an already existing derivation goal.
*/
void addWantedOutputs(const OutputsSpec & outputs);
/* The states. */
/**
* The states.
*/
void getDerivation();
void loadDerivation();
void haveDerivation();
@ -159,28 +206,42 @@ struct DerivationGoal : public Goal
void resolvedFinished();
/* Is the build hook willing to perform the build? */
/**
* Is the build hook willing to perform the build?
*/
HookReply tryBuildHook();
virtual int getChildStatus();
/* Check that the derivation outputs all exist and register them
as valid. */
/**
* Check that the derivation outputs all exist and register them
* as valid.
*/
virtual DrvOutputs registerOutputs();
/* Open a log file and a pipe to it. */
/**
* Open a log file and a pipe to it.
*/
Path openLogFile();
/* Sign the newly built realisation if the store allows it */
/**
* Sign the newly built realisation if the store allows it
*/
virtual void signRealisation(Realisation&) {}
/* Close the log file. */
/**
* Close the log file.
*/
void closeLogFile();
/* Close the read side of the logger pipe. */
/**
* Close the read side of the logger pipe.
*/
virtual void closeReadPipes();
/* Cleanup hooks for buildDone() */
/**
* Cleanup hooks for buildDone()
*/
virtual void cleanupHookFinally();
virtual void cleanupPreChildKill();
virtual void cleanupPostChildKill();
@ -190,30 +251,40 @@ struct DerivationGoal : public Goal
virtual bool isReadDesc(int fd);
/* Callback used by the worker to write to the log. */
/**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;
void flushLine();
/* Wrappers around the corresponding Store methods that first consult the
derivation. This is currently needed because when there is no drv file
there also is no DB entry. */
/**
* Wrappers around the corresponding Store methods that first consult the
* derivation. This is currently needed because when there is no drv file
* there also is no DB entry.
*/
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
OutputPathMap queryDerivationOutputMap();
/* Update 'initialOutputs' to determine the current status of the
outputs of the derivation. Also returns a Boolean denoting
whether all outputs are valid and non-corrupt, and a
'DrvOutputs' structure containing the valid and wanted
outputs. */
/**
* Update 'initialOutputs' to determine the current status of the
* outputs of the derivation. Also returns a Boolean denoting
* whether all outputs are valid and non-corrupt, and a
* 'DrvOutputs' structure containing the valid and wanted
* outputs.
*/
std::pair<bool, DrvOutputs> checkPathValidity();
/* Aborts if any output is not valid or corrupt, and otherwise
returns a 'DrvOutputs' structure containing the wanted
outputs. */
/**
* Aborts if any output is not valid or corrupt, and otherwise
* returns a 'DrvOutputs' structure containing the wanted
* outputs.
*/
DrvOutputs assertPathValidity();
/* Forcibly kill the child process, if any. */
/**
* Forcibly kill the child process, if any.
*/
virtual void killChild();
void repairClosure();

35
src/libstore/build/drv-output-substitution-goal.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "store-api.hh"
#include "goal.hh"
@ -10,24 +11,34 @@ namespace nix {
class Worker;
// Substitution of a derivation output.
// This is done in three steps:
// 1. Fetch the output info from a substituter
// 2. Substitute the corresponding output path
// 3. Register the output info
/**
* Substitution of a derivation output.
* This is done in three steps:
* 1. Fetch the output info from a substituter
* 2. Substitute the corresponding output path
* 3. Register the output info
*/
class DrvOutputSubstitutionGoal : public Goal {
// The drv output we're trying to substitue
/**
* The drv output we're trying to substitue
*/
DrvOutput id;
// The realisation corresponding to the given output id.
// Will be filled once we can get it.
/**
* The realisation corresponding to the given output id.
* Will be filled once we can get it.
*/
std::shared_ptr<const Realisation> outputInfo;
/* The remaining substituters. */
/**
* The remaining substituters.
*/
std::list<ref<Store>> subs;
/* The current substituter. */
/**
* The current substituter.
*/
std::shared_ptr<Store> sub;
struct DownloadState
@ -38,7 +49,9 @@ class DrvOutputSubstitutionGoal : public Goal {
std::shared_ptr<DownloadState> downloadState;
/* Whether a substituter failed. */
/**
* Whether a substituter failed.
*/
bool substituterFailed = false;
public:

71
src/libstore/build/goal.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "types.hh"
#include "store-api.hh"
@ -6,11 +7,15 @@
namespace nix {
/* Forward definition. */
/**
* Forward definition.
*/
struct Goal;
class Worker;
/* A pointer to a goal. */
/**
* A pointer to a goal.
*/
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
@ -18,48 +23,72 @@ struct CompareGoalPtrs {
bool operator() (const GoalPtr & a, const GoalPtr & b) const;
};
/* Set of goals. */
/**
* Set of goals.
*/
typedef std::set<GoalPtr, CompareGoalPtrs> Goals;
typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
/* A map of paths to goals (and the other way around). */
/**
* A map of paths to goals (and the other way around).
*/
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
struct Goal : public std::enable_shared_from_this<Goal>
{
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
/* Backlink to the worker. */
/**
* Backlink to the worker.
*/
Worker & worker;
/* Goals that this goal is waiting for. */
/**
* Goals that this goal is waiting for.
*/
Goals waitees;
/* Goals waiting for this one to finish. Must use weak pointers
here to prevent cycles. */
/**
* Goals waiting for this one to finish. Must use weak pointers
* here to prevent cycles.
*/
WeakGoals waiters;
/* Number of goals we are/were waiting for that have failed. */
/**
* Number of goals we are/were waiting for that have failed.
*/
size_t nrFailed = 0;
/* Number of substitution goals we are/were waiting for that
failed because there are no substituters. */
/**
* Number of substitution goals we are/were waiting for that
* failed because there are no substituters.
*/
size_t nrNoSubstituters = 0;
/* Number of substitution goals we are/were waiting for that
failed because they had unsubstitutable references. */
/**
* Number of substitution goals we are/were waiting for that
* failed because they had unsubstitutable references.
*/
size_t nrIncompleteClosure = 0;
/* Name of this goal for debugging purposes. */
/**
* Name of this goal for debugging purposes.
*/
std::string name;
/* Whether the goal is finished. */
/**
* Whether the goal is finished.
*/
ExitCode exitCode = ecBusy;
/* Build result. */
/**
* Build result.
*/
BuildResult buildResult;
/* Exception containing an error message, if any. */
/**
* Exception containing an error message, if any.
*/
std::optional<Error> ex;
Goal(Worker & worker, DerivedPath path)
@ -95,9 +124,11 @@ struct Goal : public std::enable_shared_from_this<Goal>
return name;
}
/* Callback in case of a timeout. It should wake up its waiters,
get rid of any running child processes that are being monitored
by the worker (important!), etc. */
/**
* Callback in case of a timeout. It should wake up its waiters,
* get rid of any running child processes that are being monitored
* by the worker (important!), etc.
*/
virtual void timedOut(Error && ex) = 0;
virtual std::string key() = 0;

17
src/libstore/build/hook-instance.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "logging.hh"
#include "serialise.hh"
@ -7,16 +8,24 @@ namespace nix {
struct HookInstance
{
/* Pipes for talking to the build hook. */
/**
* Pipes for talking to the build hook.
*/
Pipe toHook;
/* Pipe for the hook's standard output/error. */
/**
* Pipe for the hook's standard output/error.
*/
Pipe fromHook;
/* Pipe for the builder's standard output/error. */
/**
* Pipe for the builder's standard output/error.
*/
Pipe builderOut;
/* The process ID of the hook. */
/**
* The process ID of the hook.
*/
Pid pid;
FdSink sink;

5
src/libstore/build/local-derivation-goal.cc
View file

@ -1415,6 +1415,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
virtual void addBuildLog(const StorePath & path, std::string_view log) override
{ unsupported("addBuildLog"); }
std::optional<TrustedFlag> isTrustedClient() override
{ return NotTrusted; }
};
@ -1467,7 +1470,7 @@ void LocalDerivationGoal::startDaemon()
FdSink to(remote.get());
try {
daemon::processConnection(store, from, to,
daemon::NotTrusted, daemon::Recursive);
NotTrusted, daemon::Recursive);
debug("terminated daemon connection");
} catch (SysError &) {
ignoreException();

222
src/libstore/build/local-derivation-goal.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "derivation-goal.hh"
#include "local-store.hh"
@ -9,49 +10,75 @@ struct LocalDerivationGoal : public DerivationGoal
{
LocalStore & getLocalStore();
/* User selected for running the builder. */
/**
* User selected for running the builder.
*/
std::unique_ptr<UserLock> buildUser;
/* The process ID of the builder. */
/**
* The process ID of the builder.
*/
Pid pid;
/* The cgroup of the builder, if any. */
/**
* The cgroup of the builder, if any.
*/
std::optional<Path> cgroup;
/* The temporary directory. */
/**
* The temporary directory.
*/
Path tmpDir;
/* The path of the temporary directory in the sandbox. */
/**
* The path of the temporary directory in the sandbox.
*/
Path tmpDirInSandbox;
/* Master side of the pseudoterminal used for the builder's
standard output/error. */
/**
* Master side of the pseudoterminal used for the builder's
* standard output/error.
*/
AutoCloseFD builderOut;
/* Pipe for synchronising updates to the builder namespaces. */
/**
* Pipe for synchronising updates to the builder namespaces.
*/
Pipe userNamespaceSync;
/* The mount namespace and user namespace of the builder, used to add additional
paths to the sandbox as a result of recursive Nix calls. */
/**
* The mount namespace and user namespace of the builder, used to add additional
* paths to the sandbox as a result of recursive Nix calls.
*/
AutoCloseFD sandboxMountNamespace;
AutoCloseFD sandboxUserNamespace;
/* On Linux, whether we're doing the build in its own user
namespace. */
/**
* On Linux, whether we're doing the build in its own user
* namespace.
*/
bool usingUserNamespace = true;
/* Whether we're currently doing a chroot build. */
/**
* Whether we're currently doing a chroot build.
*/
bool useChroot = false;
Path chrootRootDir;
/* RAII object to delete the chroot directory. */
/**
* RAII object to delete the chroot directory.
*/
std::shared_ptr<AutoDelete> autoDelChroot;
/* Whether to run the build in a private network namespace. */
/**
* Whether to run the build in a private network namespace.
*/
bool privateNetwork = false;
/* Stuff we need to pass to initChild(). */
/**
* Stuff we need to pass to initChild().
*/
struct ChrootPath {
Path source;
bool optional;
@ -70,30 +97,35 @@ struct LocalDerivationGoal : public DerivationGoal
SandboxProfile additionalSandboxProfile;
#endif
/* Hash rewriting. */
/**
* Hash rewriting.
*/
StringMap inputRewrites, outputRewrites;
typedef map<StorePath, StorePath> RedirectedOutputs;
RedirectedOutputs redirectedOutputs;
/* The outputs paths used during the build.
- Input-addressed derivations or fixed content-addressed outputs are
sometimes built when some of their outputs already exist, and can not
be hidden via sandboxing. We use temporary locations instead and
rewrite after the build. Otherwise the regular predetermined paths are
put here.
- Floating content-addressed derivations do not know their final build
output paths until the outputs are hashed, so random locations are
used, and then renamed. The randomness helps guard against hidden
self-references.
/**
* The outputs paths used during the build.
*
* - Input-addressed derivations or fixed content-addressed outputs are
* sometimes built when some of their outputs already exist, and can not
* be hidden via sandboxing. We use temporary locations instead and
* rewrite after the build. Otherwise the regular predetermined paths are
* put here.
*
* - Floating content-addressed derivations do not know their final build
* output paths until the outputs are hashed, so random locations are
* used, and then renamed. The randomness helps guard against hidden
* self-references.
*/
OutputPathMap scratchOutputs;
/* Path registration info from the previous round, if we're
building multiple times. Since this contains the hash, it
allows us to compare whether two rounds produced the same
result. */
/**
* Path registration info from the previous round, if we're
* building multiple times. Since this contains the hash, it
* allows us to compare whether two rounds produced the same
* result.
*/
std::map<Path, ValidPathInfo> prevInfos;
uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
@ -101,25 +133,37 @@ struct LocalDerivationGoal : public DerivationGoal
const static Path homeDir;
/* The recursive Nix daemon socket. */
/**
* The recursive Nix daemon socket.
*/
AutoCloseFD daemonSocket;
/* The daemon main thread. */
/**
* The daemon main thread.
*/
std::thread daemonThread;
/* The daemon worker threads. */
/**
* The daemon worker threads.
*/
std::vector<std::thread> daemonWorkerThreads;
/* Paths that were added via recursive Nix calls. */
/**
* Paths that were added via recursive Nix calls.
*/
StorePathSet addedPaths;
/* Realisations that were added via recursive Nix calls. */
/**
* Realisations that were added via recursive Nix calls.
*/
std::set<DrvOutput> addedDrvOutputs;
/* Recursive Nix calls are only allowed to build or realize paths
in the original input closure or added via a recursive Nix call
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
/nix/store/<bla> is some arbitrary path in a binary cache). */
/**
* Recursive Nix calls are only allowed to build or realize paths
* in the original input closure or added via a recursive Nix call
* (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
* /nix/store/<bla> is some arbitrary path in a binary cache).
*/
bool isAllowed(const StorePath & path)
{
return inputPaths.count(path) || addedPaths.count(path);
@ -137,55 +181,81 @@ struct LocalDerivationGoal : public DerivationGoal
virtual ~LocalDerivationGoal() override;
/* Whether we need to perform hash rewriting if there are valid output paths. */
/**
* Whether we need to perform hash rewriting if there are valid output paths.
*/
bool needsHashRewrite();
/* The additional states. */
/**
* The additional states.
*/
void tryLocalBuild() override;
/* Start building a derivation. */
/**
* Start building a derivation.
*/
void startBuilder();
/* Fill in the environment for the builder. */
/**
* Fill in the environment for the builder.
*/
void initEnv();
/* Setup tmp dir location. */
/**
* Setup tmp dir location.
*/
void initTmpDir();
/* Write a JSON file containing the derivation attributes. */
/**
* Write a JSON file containing the derivation attributes.
*/
void writeStructuredAttrs();
void startDaemon();
void stopDaemon();
/* Add 'path' to the set of paths that may be referenced by the
outputs, and make it appear in the sandbox. */
/**
* Add 'path' to the set of paths that may be referenced by the
* outputs, and make it appear in the sandbox.
*/
void addDependency(const StorePath & path);
/* Make a file owned by the builder. */
/**
* Make a file owned by the builder.
*/
void chownToBuilder(const Path & path);
int getChildStatus() override;
/* Run the builder's process. */
/**
* Run the builder's process.
*/
void runChild();
/* Check that the derivation outputs all exist and register them
as valid. */
/**
* Check that the derivation outputs all exist and register them
* as valid.
*/
DrvOutputs registerOutputs() override;
void signRealisation(Realisation &) override;
/* Check that an output meets the requirements specified by the
'outputChecks' attribute (or the legacy
'{allowed,disallowed}{References,Requisites}' attributes). */
/**
* Check that an output meets the requirements specified by the
* 'outputChecks' attribute (or the legacy
* '{allowed,disallowed}{References,Requisites}' attributes).
*/
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
/* Close the read side of the logger pipe. */
/**
* Close the read side of the logger pipe.
*/
void closeReadPipes() override;
/* Cleanup hooks for buildDone() */
/**
* Cleanup hooks for buildDone()
*/
void cleanupHookFinally() override;
void cleanupPreChildKill() override;
void cleanupPostChildKill() override;
@ -195,24 +265,36 @@ struct LocalDerivationGoal : public DerivationGoal
bool isReadDesc(int fd) override;
/* Delete the temporary directory, if we have one. */
/**
* Delete the temporary directory, if we have one.
*/
void deleteTmpDir(bool force);
/* Forcibly kill the child process, if any. */
/**
* Forcibly kill the child process, if any.
*/
void killChild() override;
/* Kill any processes running under the build user UID or in the
cgroup of the build. */
/**
* Kill any processes running under the build user UID or in the
* cgroup of the build.
*/
void killSandbox(bool getStats);
/* Create alternative path calculated from but distinct from the
input, so we can avoid overwriting outputs (or other store paths)
that already exist. */
/**
* Create alternative path calculated from but distinct from the
* input, so we can avoid overwriting outputs (or other store paths)
* that already exist.
*/
StorePath makeFallbackPath(const StorePath & path);
/* Make a path to another based on the output name along with the
derivation hash. */
/* FIXME add option to randomize, so we can audit whether our
rewrites caught everything */
/**
* Make a path to another based on the output name along with the
* derivation hash.
*
* @todo Add option to randomize, so we can audit whether our
* rewrites caught everything
*/
StorePath makeFallbackPath(std::string_view outputName);
};

1
src/libstore/build/personality.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include <string>

63
src/libstore/build/substitution-goal.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "lock.hh"
#include "store-api.hh"
@ -10,38 +11,58 @@ class Worker;
struct PathSubstitutionGoal : public Goal
{
/* The store path that should be realised through a substitute. */
/**
* The store path that should be realised through a substitute.
*/
StorePath storePath;
/* The path the substituter refers to the path as. This will be
different when the stores have different names. */
/**
* The path the substituter refers to the path as. This will be
* different when the stores have different names.
*/
std::optional<StorePath> subPath;
/* The remaining substituters. */
/**
* The remaining substituters.
*/
std::list<ref<Store>> subs;
/* The current substituter. */
/**
* The current substituter.
*/
std::shared_ptr<Store> sub;
/* Whether a substituter failed. */
/**
* Whether a substituter failed.
*/
bool substituterFailed = false;
/* Path info returned by the substituter's query info operation. */
/**
* Path info returned by the substituter's query info operation.
*/
std::shared_ptr<const ValidPathInfo> info;
/* Pipe for the substituter's standard output. */
/**
* Pipe for the substituter's standard output.
*/
Pipe outPipe;
/* The substituter thread. */
/**
* The substituter thread.
*/
std::thread thr;
std::promise<void> promise;
/* Whether to try to repair a valid path. */
/**
* Whether to try to repair a valid path.
*/
RepairFlag repair;
/* Location where we're downloading the substitute. Differs from
storePath when doing a repair. */
/**
* Location where we're downloading the substitute. Differs from
* storePath when doing a repair.
*/
Path destPath;
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
@ -50,7 +71,9 @@ struct PathSubstitutionGoal : public Goal
typedef void (PathSubstitutionGoal::*GoalState)();
GoalState state;
/* Content address for recomputing store path */
/**
* Content address for recomputing store path
*/
std::optional<ContentAddress> ca;
void done(
@ -64,16 +87,20 @@ public:
void timedOut(Error && ex) override { abort(); };
/**
* We prepend "a$" to the key name to ensure substitution goals
* happen before derivation goals.
*/
std::string key() override
{
/* "a$" ensures substitution goals happen before derivation
goals. */
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
}
void work() override;
/* The states. */
/**
* The states.
*/
void init();
void tryNext();
void gotInfo();
@ -81,7 +108,9 @@ public:
void tryToRun();
void finished();
/* Callback used by the worker to write to the log. */
/**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;

187
src/libstore/build/worker.hh
View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "types.hh"
#include "lock.hh"
@ -16,24 +17,29 @@ struct DerivationGoal;
struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal;
/* Workaround for not being able to declare a something like
class PathSubstitutionGoal : public Goal;
even when Goal is a complete type.
This is still a static cast. The purpose of exporting it is to define it in
a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
is opaque. */
/**
* Workaround for not being able to declare a something like
*
* ```c++
* class PathSubstitutionGoal : public Goal;
* ```
* even when Goal is a complete type.
*
* This is still a static cast. The purpose of exporting it is to define it in
* a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
* is opaque.
*/
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/* A mapping used to remember for each child process to what goal it
belongs, and file descriptors for receiving log data and output
path creation commands. */
/**
* A mapping used to remember for each child process to what goal it
* belongs, and file descriptors for receiving log data and output
* path creation commands.
*/
struct Child
{
WeakGoalPtr goal;
@ -41,14 +47,19 @@ struct Child
std::set<int> fds;
bool respectTimeouts;
bool inBuildSlot;
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
/**
* Time we last got output on stdout/stderr
*/
steady_time_point lastOutput;
steady_time_point timeStarted;
};
/* Forward definition. */
struct HookInstance;
/* The worker class. */
/**
* The worker class.
*/
class Worker
{
private:
@ -56,38 +67,58 @@ private:
/* Note: the worker should only have strong pointers to the
top-level goals. */
/* The top-level goals of the worker. */
/**
* The top-level goals of the worker.
*/
Goals topGoals;
/* Goals that are ready to do some work. */
/**
* Goals that are ready to do some work.
*/
WeakGoals awake;
/* Goals waiting for a build slot. */
/**
* Goals waiting for a build slot.
*/
WeakGoals wantingToBuild;
/* Child processes currently running. */
/**
* Child processes currently running.
*/
std::list<Child> children;
/* Number of build slots occupied. This includes local builds and
substitutions but not remote builds via the build hook. */
/**
* Number of build slots occupied. This includes local builds and
* substitutions but not remote builds via the build hook.
*/
unsigned int nrLocalBuilds;
/* Maps used to prevent multiple instantiations of a goal for the
same derivation / path. */
/**
* Maps used to prevent multiple instantiations of a goal for the
* same derivation / path.
*/
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
/* Goals waiting for busy paths to be unlocked. */
/**
* Goals waiting for busy paths to be unlocked.
*/
WeakGoals waitingForAnyGoal;
/* Goals sleeping for a few seconds (polling a lock). */
/**
* Goals sleeping for a few seconds (polling a lock).
*/
WeakGoals waitingForAWhile;
/* Last time the goals in `waitingForAWhile' where woken up. */
/**
* Last time the goals in `waitingForAWhile` where woken up.
*/
steady_time_point lastWokenUp;
/* Cache for pathContentsGood(). */
/**
* Cache for pathContentsGood().
*/
std::map<StorePath, bool> pathContentsGoodCache;
public:
@ -96,17 +127,25 @@ public:
const Activity actDerivations;
const Activity actSubstitutions;
/* Set if at least one derivation had a BuildError (i.e. permanent
failure). */
/**
* Set if at least one derivation had a BuildError (i.e. permanent
* failure).
*/
bool permanentFailure;
/* Set if at least one derivation had a timeout. */
/**
* Set if at least one derivation had a timeout.
*/
bool timedOut;
/* Set if at least one derivation fails with a hash mismatch. */
/**
* Set if at least one derivation fails with a hash mismatch.
*/
bool hashMismatch;
/* Set if at least one derivation is not deterministic in check mode. */
/**
* Set if at least one derivation is not deterministic in check mode.
*/
bool checkMismatch;
Store & store;
@ -128,16 +167,22 @@ public:
uint64_t expectedNarSize = 0;
uint64_t doneNarSize = 0;
/* Whether to ask the build hook if it can build a derivation. If
it answers with "decline-permanently", we don't try again. */
/**
* Whether to ask the build hook if it can build a derivation. If
* it answers with "decline-permanently", we don't try again.
*/
bool tryBuildHook = true;
Worker(Store & store, Store & evalStore);
~Worker();
/* Make a goal (with caching). */
/**
* Make a goal (with caching).
*/
/* derivation goal */
/**
* derivation goal
*/
private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const OutputsSpec & wantedOutputs,
@ -150,56 +195,80 @@ public:
const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */
/**
* substitution goal
*/
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
/**
* Remove a dead goal.
*/
void removeGoal(GoalPtr goal);
/* Wake up a goal (i.e., there is something for it to do). */
/**
* Wake up a goal (i.e., there is something for it to do).
*/
void wakeUp(GoalPtr goal);
/* Return the number of local build and substitution processes
currently running (but not remote builds via the build
hook). */
/**
* Return the number of local build and substitution processes
* currently running (but not remote builds via the build
* hook).
*/
unsigned int getNrLocalBuilds();
/* Registers a running child process. `inBuildSlot' means that
the process counts towards the jobs limit. */
/**
* Registers a running child process. `inBuildSlot` means that
* the process counts towards the jobs limit.
*/
void childStarted(GoalPtr goal, const std::set<int> & fds,
bool inBuildSlot, bool respectTimeouts);
/* Unregisters a running child process. `wakeSleepers' should be
false if there is no sense in waking up goals that are sleeping
because they can't run yet (e.g., there is no free build slot,
or the hook would still say `postpone'). */
/**
* Unregisters a running child process. `wakeSleepers` should be
* false if there is no sense in waking up goals that are sleeping
* because they can't run yet (e.g., there is no free build slot,
* or the hook would still say `postpone`).
*/
void childTerminated(Goal * goal, bool wakeSleepers = true);
/* Put `goal' to sleep until a build slot becomes available (which
might be right away). */
/**
* Put `goal` to sleep until a build slot becomes available (which
* might be right away).
*/
void waitForBuildSlot(GoalPtr goal);
/* Wait for any goal to finish. Pretty indiscriminate way to
wait for some resource that some other goal is holding. */
/**
* Wait for any goal to finish. Pretty indiscriminate way to
* wait for some resource that some other goal is holding.
*/
void waitForAnyGoal(GoalPtr goal);
/* Wait for a few seconds and then retry this goal. Used when
waiting for a lock held by another process. This kind of
polling is inefficient, but POSIX doesn't really provide a way
to wait for multiple locks in the main select() loop. */
/**
* Wait for a few seconds and then retry this goal. Used when
* waiting for a lock held by another process. This kind of
* polling is inefficient, but POSIX doesn't really provide a way
* to wait for multiple locks in the main select() loop.
*/
void waitForAWhile(GoalPtr goal);
/* Loop until the specified top-level goals have finished. */
/**
* Loop until the specified top-level goals have finished.
*/
void run(const Goals & topGoals);
/* Wait for input to become available. */
/**
* Wait for input to become available.
*/
void waitForInput();
unsigned int exitStatus();
/* Check whether the given valid path exists and has the right
contents. */
/**
* Check whether the given valid path exists and has the right
* contents.
*/
bool pathContentsGood(const StorePath & path);
void markContentsGood(const StorePath & path);