nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-03 23:51:00 +01:00
Code Activity

Merge commit 'aa99005004' into ca-drv-exotic

Browse source
This commit is contained in:
John Ericson 2023-04-01 15:15:32 -04:00
commit f7f44f7c96
315 changed files with 6194 additions and 3610 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/libstore/build/derivation-goal.cc
View file

@ -199,10 +199,10 @@ void DerivationGoal::haveDerivation()
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (!drv->type().hasKnownOutputPaths())
settings.requireExperimentalFeature(Xp::CaDerivations);
experimentalFeatureSettings.require(Xp::CaDerivations);
if (!drv->type().isPure()) {
settings.requireExperimentalFeature(Xp::ImpureDerivations);
experimentalFeatureSettings.require(Xp::ImpureDerivations);
for (auto & [outputName, output] : drv->outputs) {
auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
@ -336,7 +336,7 @@ void DerivationGoal::gaveUpOnSubstitution()
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
/* Ensure that pure, non-fixed-output derivations don't
depend on impure derivations. */
if (settings.isExperimentalFeatureEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
auto inputDrv = worker.evalStore.readDerivation(i.first);
if (!inputDrv.type().isPure())
throw Error("pure derivation '%s' depends on impure derivation '%s'",
@ -477,7 +477,7 @@ void DerivationGoal::inputsRealised()
ca.fixed
/* Can optionally resolve if fixed, which is good
for avoiding unnecessary rebuilds. */
? settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
? experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
/* Must resolve if floating and there are any inputs
drvs. */
: true);
@ -488,7 +488,7 @@ void DerivationGoal::inputsRealised()
}, drvType.raw());
if (resolveDrv && !fullDrv.inputDrvs.empty()) {
settings.requireExperimentalFeature(Xp::CaDerivations);
experimentalFeatureSettings.require(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a
@ -732,7 +732,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
tmpPath (the replacement), so we have to move it out of the
way first. We'd better not be interrupted here, because if
we're repairing (say) Glibc, we end up with a broken system. */
Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), random());
if (pathExists(storePath))
movePath(storePath, oldPath);
@ -1352,7 +1352,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
};
}
auto drvOutput = DrvOutput{info.outputHash, i.first};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = {
.path = real->outPath,

23
src/libstore/build/drv-output-substitution-goal.cc
View file

@ -61,20 +61,25 @@ void DrvOutputSubstitutionGoal::tryNext()
// FIXME: Make async
// outputInfo = sub->queryRealisation(id);
outPipe.create();
promise = decltype(promise)();
/* The callback of the curl download below can outlive `this` (if
some other error occurs), so it must not touch `this`. So put
the shared state in a separate refcounted object. */
downloadState = std::make_shared<DownloadState>();
downloadState->outPipe.create();
sub->queryRealisation(
id, { [&](std::future<std::shared_ptr<const Realisation>> res) {
id,
{ [downloadState(downloadState)](std::future<std::shared_ptr<const Realisation>> res) {
try {
Finally updateStats([this]() { outPipe.writeSide.close(); });
promise.set_value(res.get());
Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); });
downloadState->promise.set_value(res.get());
} catch (...) {
promise.set_exception(std::current_exception());
downloadState->promise.set_exception(std::current_exception());
}
} });
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
worker.childStarted(shared_from_this(), {downloadState->outPipe.readSide.get()}, true, false);
state = &DrvOutputSubstitutionGoal::realisationFetched;
}
@ -84,7 +89,7 @@ void DrvOutputSubstitutionGoal::realisationFetched()
worker.childTerminated(this);
try {
outputInfo = promise.get_future().get();
outputInfo = downloadState->promise.get_future().get();
} catch (std::exception & e) {
printError(e.what());
substituterFailed = true;
@ -155,7 +160,7 @@ void DrvOutputSubstitutionGoal::work()
void DrvOutputSubstitutionGoal::handleEOF(int fd)
{
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}

12
src/libstore/build/drv-output-substitution-goal.hh
View file

@ -16,7 +16,7 @@ class Worker;
// 2. Substitute the corresponding output path
// 3. Register the output info
class DrvOutputSubstitutionGoal : public Goal {
private:
// The drv output we're trying to substitue
DrvOutput id;
@ -30,9 +30,13 @@ private:
/* The current substituter. */
std::shared_ptr<Store> sub;
Pipe outPipe;
std::thread thr;
std::promise<std::shared_ptr<const Realisation>> promise;
struct DownloadState
{
Pipe outPipe;
std::promise<std::shared_ptr<const Realisation>> promise;
};
std::shared_ptr<DownloadState> downloadState;
/* Whether a substituter failed. */
bool substituterFailed = false;

4
src/libstore/build/goal.cc
View file

@ -78,9 +78,9 @@ void Goal::amDone(ExitCode result, std::optional<Error> ex)
}
void Goal::trace(const FormatOrString & fs)
void Goal::trace(std::string_view s)
{
debug("%1%: %2%", name, fs.s);
debug("%1%: %2%", name, s);
}
}

2
src/libstore/build/goal.hh
View file

@ -88,7 +88,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
abort();
}
void trace(const FormatOrString & fs);
void trace(std::string_view s);
std::string getName()
{

5
src/libstore/build/hook-instance.cc
View file

@ -35,7 +35,10 @@ HookInstance::HookInstance()
/* Fork the hook. */
pid = startProcess([&]() {
commonChildInit(fromHook);
if (dup2(fromHook.writeSide.get(), STDERR_FILENO) == -1)
throw SysError("cannot pipe standard error into log file");
commonChildInit();
if (chdir("/") == -1) throw SysError("changing into /");

104
src/libstore/build/local-derivation-goal.cc
View file

@ -292,7 +292,7 @@ void LocalDerivationGoal::closeReadPipes()
if (hook) {
DerivationGoal::closeReadPipes();
} else
builderOut.readSide = -1;
builderOut.close();
}
@ -413,7 +413,7 @@ void LocalDerivationGoal::startBuilder()
)
{
#if __linux__
settings.requireExperimentalFeature(Xp::Cgroups);
experimentalFeatureSettings.require(Xp::Cgroups);
auto cgroupFS = getCgroupFS();
if (!cgroupFS)
@ -650,7 +650,7 @@ void LocalDerivationGoal::startBuilder()
/* Clean up the chroot directory automatically. */
autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootRootDir);
// FIXME: make this 0700
if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1)
@ -753,8 +753,7 @@ void LocalDerivationGoal::startBuilder()
throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
printMsg(lvlChatty, format("executing pre-build hook '%1%'")
% settings.preBuildHook);
printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook);
auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
Strings({ worker.store.printStorePath(drvPath) });
enum BuildHookState {
@ -803,15 +802,13 @@ void LocalDerivationGoal::startBuilder()
/* Create the log file. */
Path logFile = openLogFile();
/* Create a pipe to get the output of the builder. */
//builderOut.create();
builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
if (!builderOut.readSide)
/* Create a pseudoterminal to get the output of the builder. */
builderOut = posix_openpt(O_RDWR | O_NOCTTY);
if (!builderOut)
throw SysError("opening pseudoterminal master");
// FIXME: not thread-safe, use ptsname_r
std::string slaveName(ptsname(builderOut.readSide.get()));
std::string slaveName = ptsname(builderOut.get());
if (buildUser) {
if (chmod(slaveName.c_str(), 0600))
@ -822,34 +819,34 @@ void LocalDerivationGoal::startBuilder()
}
#if __APPLE__
else {
if (grantpt(builderOut.readSide.get()))
if (grantpt(builderOut.get()))
throw SysError("granting access to pseudoterminal slave");
}
#endif
#if 0
// Mount the pt in the sandbox so that the "tty" command works.
// FIXME: this doesn't work with the new devpts in the sandbox.
if (useChroot)
dirsInChroot[slaveName] = {slaveName, false};
#endif
if (unlockpt(builderOut.readSide.get()))
if (unlockpt(builderOut.get()))
throw SysError("unlocking pseudoterminal");
builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
if (!builderOut.writeSide)
throw SysError("opening pseudoterminal slave");
/* Open the slave side of the pseudoterminal and use it as stderr. */
auto openSlave = [&]()
{
AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
if (!builderOut)
throw SysError("opening pseudoterminal slave");
// Put the pt into raw mode to prevent \n -> \r\n translation.
struct termios term;
if (tcgetattr(builderOut.writeSide.get(), &term))
throw SysError("getting pseudoterminal attributes");
// Put the pt into raw mode to prevent \n -> \r\n translation.
struct termios term;
if (tcgetattr(builderOut.get(), &term))
throw SysError("getting pseudoterminal attributes");
cfmakeraw(&term);
cfmakeraw(&term);
if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
throw SysError("putting pseudoterminal into raw mode");
if (tcsetattr(builderOut.get(), TCSANOW, &term))
throw SysError("putting pseudoterminal into raw mode");
if (dup2(builderOut.get(), STDERR_FILENO) == -1)
throw SysError("cannot pipe standard error into log file");
};
buildResult.startTime = time(0);
@ -898,7 +895,16 @@ void LocalDerivationGoal::startBuilder()
usingUserNamespace = userNamespacesSupported();
Pipe sendPid;
sendPid.create();
Pid helper = startProcess([&]() {
sendPid.readSide.close();
/* We need to open the slave early, before
CLONE_NEWUSER. Otherwise we get EPERM when running as
root. */
openSlave();
/* Drop additional groups here because we can't do it
after we've created the new user namespace. FIXME:
@ -920,11 +926,12 @@ void LocalDerivationGoal::startBuilder()
pid_t child = startProcess([&]() { runChild(); }, options);
writeFull(builderOut.writeSide.get(),
fmt("%d %d\n", usingUserNamespace, child));
writeFull(sendPid.writeSide.get(), fmt("%d\n", child));
_exit(0);
});
sendPid.writeSide.close();
if (helper.wait() != 0)
throw Error("unable to start build process");
@ -936,10 +943,9 @@ void LocalDerivationGoal::startBuilder()
userNamespaceSync.writeSide = -1;
});
auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
assert(ss.size() == 2);
usingUserNamespace = ss[0] == "1";
pid = string2Int<pid_t>(ss[1]).value();
auto ss = tokenizeString<std::vector<std::string>>(readLine(sendPid.readSide.get()));
assert(ss.size() == 1);
pid = string2Int<pid_t>(ss[0]).value();
if (usingUserNamespace) {
/* Set the UID/GID mapping of the builder's user namespace
@ -994,21 +1000,21 @@ void LocalDerivationGoal::startBuilder()
#endif
{
pid = startProcess([&]() {
openSlave();
runChild();
});
}
/* parent */
pid.setSeparatePG(true);
builderOut.writeSide = -1;
worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
worker.childStarted(shared_from_this(), {builderOut.get()}, true, true);
/* Check if setting up the build environment failed. */
std::vector<std::string> msgs;
while (true) {
std::string msg = [&]() {
try {
return readLine(builderOut.readSide.get());
return readLine(builderOut.get());
} catch (Error & e) {
auto status = pid.wait();
e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)",
@ -1020,7 +1026,7 @@ void LocalDerivationGoal::startBuilder()
}();
if (msg.substr(0, 1) == "\2") break;
if (msg.substr(0, 1) == "\1") {
FdSource source(builderOut.readSide.get());
FdSource source(builderOut.get());
auto ex = readError(source);
ex.addTrace({}, "while setting up the build environment");
throw ex;
@ -1104,7 +1110,7 @@ void LocalDerivationGoal::initEnv()
env["NIX_STORE"] = worker.store.storeDir;
/* The maximum number of cores to utilize for parallel building. */
env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores);
initTmpDir();
@ -1155,10 +1161,10 @@ void LocalDerivationGoal::writeStructuredAttrs()
writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
chownToBuilder(tmpDir + "/.attrs.sh");
env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh";
env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh";
writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
chownToBuilder(tmpDir + "/.attrs.json");
env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json";
env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json";
}
}
@ -1414,7 +1420,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
void LocalDerivationGoal::startDaemon()
{
settings.requireExperimentalFeature(Xp::RecursiveNix);
experimentalFeatureSettings.require(Xp::RecursiveNix);
Store::Params params;
params["path-info-cache-size"] = "0";
@ -1650,7 +1656,7 @@ void LocalDerivationGoal::runChild()
try { /* child */
commonChildInit(builderOut);
commonChildInit();
try {
setupSeccomp();
@ -2063,7 +2069,7 @@ void LocalDerivationGoal::runChild()
/* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
Path globalTmpDir = canonPath(getEnvNonEmpty("TMPDIR").value_or("/tmp"), true);
/* They don't like trailing slashes on subpath directives */
if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
@ -2274,7 +2280,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
bool discardReferences = false;
if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) {
settings.requireExperimentalFeature(Xp::DiscardReferences);
experimentalFeatureSettings.require(Xp::DiscardReferences);
if (auto output = get(*udr, outputName)) {
if (!output->is_boolean())
throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string());
@ -2698,7 +2704,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
},
.outPath = newInfo.path
};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
&& drv->type().isPure())
{
signRealisation(thisRealisation);
@ -2896,7 +2902,7 @@ void LocalDerivationGoal::deleteTmpDir(bool force)
bool LocalDerivationGoal::isReadDesc(int fd)
{
return (hook && DerivationGoal::isReadDesc(fd)) ||
(!hook && fd == builderOut.readSide.get());
(!hook && fd == builderOut.get());
}

5
src/libstore/build/local-derivation-goal.hh
View file

@ -24,8 +24,9 @@ struct LocalDerivationGoal : public DerivationGoal
/* The path of the temporary directory in the sandbox. */
Path tmpDirInSandbox;
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
/* Master side of the pseudoterminal used for the builder's
standard output/error. */
AutoCloseFD builderOut;
/* Pipe for synchronising updates to the builder namespaces. */
Pipe userNamespaceSync;