1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-05 08:31:00 +01:00

More work on the scheduler for windows

- Get a rump derivation goal: hook instance will come later, local
  derivation goal will come after that.

- Start cleaning up the channel / waiting code with an abstraction.
This commit is contained in:
John Ericson 2024-05-27 17:54:02 -04:00
parent 1e2b26734b
commit bcdee80a0d
13 changed files with 331 additions and 204 deletions

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,352 @@
#pragma once
///@file
#include "parsed-derivations.hh"
#ifndef _WIN32
# include "user-lock.hh"
#endif
#include "outputs-spec.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "goal.hh"
namespace nix {
using std::map;
#ifndef _WIN32 // TODO enable build hook on Windows
struct HookInstance;
#endif
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
/**
* Unless we are repairing, we don't both to test validity and just assume it,
* so the choices are `Absent` or `Valid`.
*/
enum struct PathStatus {
Corrupt,
Absent,
Valid,
};
struct InitialOutputStatus {
StorePath path;
PathStatus status;
/**
* Valid in the store, and additionally non-corrupt if we are repairing
*/
bool isValid() const {
return status == PathStatus::Valid;
}
/**
* Merely present, allowed to be corrupt
*/
bool isPresent() const {
return status == PathStatus::Corrupt
|| status == PathStatus::Valid;
}
};
struct InitialOutput {
bool wanted;
Hash outputHash;
std::optional<InitialOutputStatus> known;
};
/**
* A goal for building some or all of the outputs of a derivation.
*/
struct DerivationGoal : public Goal
{
/**
* Whether to use an on-disk .drv file.
*/
bool useDerivation;
/** The path of the derivation. */
StorePath drvPath;
/**
* The goal for the corresponding resolved derivation
*/
std::shared_ptr<DerivationGoal> resolvedDrvGoal;
/**
* The specific outputs that we need to build.
*/
OutputsSpec wantedOutputs;
/**
* Mapping from input derivations + output names to actual store
* paths. This is filled in by waiteeDone() as each dependency
* finishes, before inputsRealised() is reached.
*/
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/**
* See `needRestart`; just for that field.
*/
enum struct NeedRestartForMoreOutputs {
/**
* The goal state machine is progressing based on the current value of
* `wantedOutputs. No actions are needed.
*/
OutputsUnmodifedDontNeed,
/**
* `wantedOutputs` has been extended, but the state machine is
* proceeding according to its old value, so we need to restart.
*/
OutputsAddedDoNeed,
/**
* The goal state machine has progressed to the point of doing a build,
* in which case all outputs will be produced, so extensions to
* `wantedOutputs` no longer require a restart.
*/
BuildInProgressWillNotNeed,
};
/**
* Whether additional wanted outputs have been added.
*/
NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
/**
* See `retrySubstitution`; just for that field.
*/
enum RetrySubstitution {
/**
* No issues have yet arose, no need to restart.
*/
NoNeed,
/**
* Something failed and there is an incomplete closure. Let's retry
* substituting.
*/
YesNeed,
/**
* We are current or have already retried substitution, and whether or
* not something goes wrong we will not retry again.
*/
AlreadyRetried,
};
/**
* Whether to retry substituting the outputs after building the
* inputs. This is done in case of an incomplete closure.
*/
RetrySubstitution retrySubstitution = RetrySubstitution::NoNeed;
/**
* The derivation stored at drvPath.
*/
std::unique_ptr<Derivation> drv;
std::unique_ptr<ParsedDerivation> parsedDrv;
/**
* The remainder is state held during the build.
*/
/**
* Locks on (fixed) output paths.
*/
PathLocks outputLocks;
/**
* All input paths (that is, the union of FS closures of the
* immediate input paths).
*/
StorePathSet inputPaths;
std::map<std::string, InitialOutput> initialOutputs;
/**
* File descriptor for the log file.
*/
AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink;
/**
* Number of bytes received from the builder's stdout/stderr.
*/
unsigned long logSize;
/**
* The most recent log lines.
*/
std::list<std::string> logTail;
std::string currentLogLine;
size_t currentLogLinePos = 0; // to handle carriage return
std::string currentHookLine;
#ifndef _WIN32 // TODO enable build hook on Windows
/**
* The build hook.
*/
std::unique_ptr<HookInstance> hook;
#endif
/**
* The sort of derivation we are building.
*/
std::optional<DerivationType> derivationType;
typedef void (DerivationGoal::*GoalState)();
GoalState state;
BuildMode buildMode;
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
std::unique_ptr<Activity> act;
/**
* Activity that denotes waiting for a lock.
*/
std::unique_ptr<Activity> actLock;
std::map<ActivityId, Activity> builderActivities;
/**
* The remote machine on which we're building.
*/
std::string machineName;
DerivationGoal(const StorePath & drvPath,
const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
virtual ~DerivationGoal();
void timedOut(Error && ex) override;
std::string key() override;
void work() override;
/**
* Add wanted outputs to an already existing derivation goal.
*/
void addWantedOutputs(const OutputsSpec & outputs);
/**
* The states.
*/
void getDerivation();
void loadDerivation();
void haveDerivation();
void outputsSubstitutionTried();
void gaveUpOnSubstitution();
void closureRepaired();
void inputsRealised();
void tryToBuild();
virtual void tryLocalBuild();
void buildDone();
void resolvedFinished();
/**
* Is the build hook willing to perform the build?
*/
HookReply tryBuildHook();
virtual int getChildStatus();
/**
* Check that the derivation outputs all exist and register them
* as valid.
*/
virtual SingleDrvOutputs registerOutputs();
/**
* Open a log file and a pipe to it.
*/
Path openLogFile();
/**
* Sign the newly built realisation if the store allows it
*/
virtual void signRealisation(Realisation&) {}
/**
* Close the log file.
*/
void closeLogFile();
/**
* Close the read side of the logger pipe.
*/
virtual void closeReadPipes();
/**
* Cleanup hooks for buildDone()
*/
virtual void cleanupHookFinally();
virtual void cleanupPreChildKill();
virtual void cleanupPostChildKill();
virtual bool cleanupDecideWhetherDiskFull();
virtual void cleanupPostOutputsRegisteredModeCheck();
virtual void cleanupPostOutputsRegisteredModeNonCheck();
virtual bool isReadDesc(Descriptor fd);
/**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(Descriptor fd, std::string_view data) override;
void handleEOF(Descriptor fd) override;
void flushLine();
/**
* Wrappers around the corresponding Store methods that first consult the
* derivation. This is currently needed because when there is no drv file
* there also is no DB entry.
*/
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
OutputPathMap queryDerivationOutputMap();
/**
* Update 'initialOutputs' to determine the current status of the
* outputs of the derivation. Also returns a Boolean denoting
* whether all outputs are valid and non-corrupt, and a
* 'SingleDrvOutputs' structure containing the valid outputs.
*/
std::pair<bool, SingleDrvOutputs> checkPathValidity();
/**
* Aborts if any output is not valid or corrupt, and otherwise
* returns a 'SingleDrvOutputs' structure containing all outputs.
*/
SingleDrvOutputs assertPathValidity();
/**
* Forcibly kill the child process, if any.
*/
virtual void killChild();
void repairClosure();
void started();
void done(
BuildResult::Status status,
SingleDrvOutputs builtOutputs = {},
std::optional<Error> ex = {});
void waiteeDone(GoalPtr waitee, ExitCode result) override;
StorePathSet exportReferences(const StorePathSet & storePaths);
JobCategory jobCategory() const override {
return JobCategory::Build;
};
};
MakeError(NotDeterministic, BuildError);
}

View file

@ -7,10 +7,7 @@
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
#ifdef _WIN32
# include "windows-async-pipe.hh"
#endif
#include "muxable-pipe.hh"
namespace nix {
@ -48,11 +45,7 @@ class DrvOutputSubstitutionGoal : public Goal {
struct DownloadState
{
#ifndef _WIN32
Pipe outPipe;
#else
windows::AsyncPipe outPipe;
#endif
MuxablePipe outPipe;
std::promise<std::shared_ptr<const Realisation>> promise;
};

View file

@ -3,10 +3,7 @@
#include "store-api.hh"
#include "goal.hh"
#ifdef _WIN32
# include "windows-async-pipe.hh"
#endif
#include "muxable-pipe.hh"
namespace nix {
@ -48,11 +45,7 @@ struct PathSubstitutionGoal : public Goal
/**
* Pipe for the substituter's standard output.
*/
#ifndef _WIN32
Pipe outPipe;
#else
windows::AsyncPipe outPipe;
#endif
MuxablePipe outPipe;
/**
* The substituter thread.

View file

@ -3,19 +3,13 @@
#include "worker.hh"
#include "substitution-goal.hh"
#include "drv-output-substitution-goal.hh"
#include "derivation-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows
# include "local-derivation-goal.hh"
# include "hook-instance.hh"
#endif
#include "signals.hh"
#ifndef _WIN32
# include <poll.h>
#else
# include <ioapiset.h>
# include "windows-error.hh"
#endif
namespace nix {
Worker::Worker(Store & store, Store & evalStore)
@ -49,7 +43,6 @@ Worker::~Worker()
assert(expectedNarSize == 0);
}
#ifndef _WIN32 // TODO Enable building on Windows
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath,
@ -73,9 +66,13 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
const OutputsSpec & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)
? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
: std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
return
#ifndef _WIN32 // TODO Enable building on Windows
dynamic_cast<LocalStore *>(&store)
? std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
:
#endif
std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
});
}
@ -83,14 +80,16 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)
? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
: std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
return
#ifndef _WIN32 // TODO Enable building on Windows
dynamic_cast<LocalStore *>(&store)
? std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
:
#endif
std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
});
}
#endif
std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
@ -122,14 +121,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr {
#ifndef _WIN32 // TODO Enable building on Windows
if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
else
throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
#else
throw UnimplementedError("Building derivations not yet implemented on Windows");
#endif
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@ -155,11 +150,9 @@ static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> &
void Worker::removeGoal(GoalPtr goal)
{
#ifndef _WIN32 // TODO Enable building on Windows
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else
#endif
if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals);
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
@ -204,7 +197,7 @@ unsigned Worker::getNrSubstitutions()
}
void Worker::childStarted(GoalPtr goal, const std::set<Child::CommChannel> & channels,
void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels,
bool inBuildSlot, bool respectTimeouts)
{
Child child;
@ -298,14 +291,12 @@ void Worker::run(const Goals & _topGoals)
for (auto & i : _topGoals) {
topGoals.insert(i);
#ifndef _WIN32 // TODO Enable building on Windows
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(goal->drvPath),
.outputs = goal->wantedOutputs,
});
} else
#endif
if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Opaque{goal->storePath});
}
@ -428,47 +419,26 @@ void Worker::waitForInput()
if (useTimeout)
vomit("sleeping %d seconds", timeout);
MuxablePipePollState state;
#ifndef _WIN32
/* Use select() to wait for the input side of any logger pipe to
become `available'. Note that `available' (i.e., non-blocking)
includes EOF. */
std::vector<struct pollfd> pollStatus;
std::map<int, size_t> fdToPollStatus;
for (auto & i : children) {
for (auto & j : i.channels) {
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
fdToPollStatus[j] = pollStatus.size() - 1;
state.pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
state.fdToPollStatus[j] = state.pollStatus.size() - 1;
}
}
if (poll(pollStatus.data(), pollStatus.size(),
useTimeout ? timeout * 1000 : -1) == -1) {
if (errno == EINTR) return;
throw SysError("waiting for input");
}
#else
OVERLAPPED_ENTRY oentries[0x20] = {0};
ULONG removed;
bool gotEOF = false;
// we are on at least Windows Vista / Server 2008 and can get many (countof(oentries)) statuses in one API call
if (!GetQueuedCompletionStatusEx(
ioport.get(),
oentries,
sizeof(oentries) / sizeof(*oentries),
&removed,
useTimeout ? timeout * 1000 : INFINITE,
false))
{
windows::WinError winError("GetQueuedCompletionStatusEx");
if (winError.lastError != WAIT_TIMEOUT)
throw winError;
assert(removed == 0);
} else {
assert(0 < removed && removed <= sizeof(oentries)/sizeof(*oentries));
}
#endif
state.poll(
#ifdef _WIN32
ioport.get(),
#endif
useTimeout ? (std::optional { timeout * 1000 }) : std::nullopt);
auto after = steady_time_point::clock::now();
/* Process all available file descriptors. FIXME: this is
@ -482,75 +452,18 @@ void Worker::waitForInput()
GoalPtr goal = j->goal.lock();
assert(goal);
#ifndef _WIN32
std::set<Descriptor> fds2(j->channels);
std::vector<unsigned char> buffer(4096);
for (auto & k : fds2) {
const auto fdPollStatusId = get(fdToPollStatus, k);
assert(fdPollStatusId);
assert(*fdPollStatusId < pollStatus.size());
if (pollStatus.at(*fdPollStatusId).revents) {
ssize_t rd = ::read(fromDescriptorReadOnly(k), buffer.data(), buffer.size());
// FIXME: is there a cleaner way to handle pt close
// than EIO? Is this even standard?
if (rd == 0 || (rd == -1 && errno == EIO)) {
debug("%1%: got EOF", goal->getName());
goal->handleEOF(k);
j->channels.erase(k);
} else if (rd == -1) {
if (errno != EINTR)
throw SysError("%s: read failed", goal->getName());
} else {
printMsg(lvlVomit, "%1%: read %2% bytes",
goal->getName(), rd);
std::string_view data((char *) buffer.data(), rd);
j->lastOutput = after;
goal->handleChildOutput(k, data);
}
}
}
#else
decltype(j->channels)::iterator p = j->channels.begin();
while (p != j->channels.end()) {
decltype(p) nextp = p;
++nextp;
for (ULONG i = 0; i < removed; i++) {
if (oentries[i].lpCompletionKey == ((ULONG_PTR)((*p)->readSide.get()) ^ 0x5555)) {
printMsg(lvlVomit, "%s: read %s bytes", goal->getName(), oentries[i].dwNumberOfBytesTransferred);
if (oentries[i].dwNumberOfBytesTransferred > 0) {
std::string data {
(char *) (*p)->buffer.data(),
oentries[i].dwNumberOfBytesTransferred,
};
//std::cerr << "read [" << data << "]" << std::endl;
j->lastOutput = after;
goal->handleChildOutput((*p)->readSide.get(), data);
}
if (gotEOF) {
debug("%s: got EOF", goal->getName());
goal->handleEOF((*p)->readSide.get());
nextp = j->channels.erase(p); // no need to maintain `j->channels` ?
} else {
BOOL rc = ReadFile((*p)->readSide.get(), (*p)->buffer.data(), (*p)->buffer.size(), &(*p)->got, &(*p)->overlapped);
if (rc) {
// here is possible (but not obligatory) to call `goal->handleChildOutput` and repeat ReadFile immediately
} else {
windows::WinError winError("ReadFile(%s, ..)", (*p)->readSide.get());
if (winError.lastError == ERROR_BROKEN_PIPE) {
debug("%s: got EOF", goal->getName());
goal->handleEOF((*p)->readSide.get());
nextp = j->channels.erase(p); // no need to maintain `j->channels` ?
} else if (winError.lastError != ERROR_IO_PENDING)
throw winError;
}
}
break;
}
}
p = nextp;
}
#endif
state.iterate(
j->channels,
[&](Descriptor k, std::string_view data) {
printMsg(lvlVomit, "%1%: read %2% bytes",
goal->getName(), data.size());
j->lastOutput = after;
goal->handleChildOutput(k, data);
},
[&](Descriptor k) {
debug("%1%: got EOF", goal->getName());
goal->handleEOF(k);
});
if (goal->exitCode == Goal::ecBusy &&
0 != settings.maxSilentTime &&

View file

@ -5,10 +5,7 @@
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
#ifdef _WIN32
# include "windows-async-pipe.hh"
#endif
#include "muxable-pipe.hh"
#include <future>
#include <thread>
@ -16,9 +13,7 @@
namespace nix {
/* Forward definition. */
#ifndef _WIN32 // TODO Enable building on Windows
struct DerivationGoal;
#endif
struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal;
@ -46,17 +41,9 @@ typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
*/
struct Child
{
using CommChannel =
#ifndef _WIN32
Descriptor
#else
windows::AsyncPipe *
#endif
;
WeakGoalPtr goal;
Goal * goal2; // ugly hackery
std::set<CommChannel> channels;
std::set<MuxablePipePollState::CommChannel> channels;
bool respectTimeouts;
bool inBuildSlot;
/**
@ -116,9 +103,7 @@ private:
* Maps used to prevent multiple instantiations of a goal for the
* same derivation / path.
*/
#ifndef _WIN32 // TODO Enable building on Windows
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
#endif
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@ -207,7 +192,6 @@ public:
* Make a goal (with caching).
*/
#ifndef _WIN32 // TODO Enable building on Windows
/**
* @ref DerivationGoal "derivation goal"
*/
@ -222,7 +206,6 @@ public:
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
#endif
/**
* @ref SubstitutionGoal "substitution goal"
@ -263,7 +246,7 @@ public:
* Registers a running child process. `inBuildSlot` means that
* the process counts towards the jobs limit.
*/
void childStarted(GoalPtr goal, const std::set<Child::CommChannel> & channels,
void childStarted(GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels,
bool inBuildSlot, bool respectTimeouts);
/**