nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-27 20:51:00 +01:00
Code Activity

Merge remote-tracking branch 'detsys/detsys-main' into logger-improvements

Browse source
This commit is contained in:
Eelco Dolstra 2025-03-26 23:21:32 +01:00
commit c8692b378b
206 changed files with 4776 additions and 2018 deletions
Download patch file Download diff file Expand all files Collapse all files

209
src/libstore/unix/build/local-derivation-goal.cc
View file

@ -184,10 +184,6 @@ void LocalDerivationGoal::killSandbox(bool getStats)
Goal::Co LocalDerivationGoal::tryLocalBuild()
{
#if __APPLE__
additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
#endif
unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= settings.maxBuildJobs) {
worker.waitForBuildSlot(shared_from_this());
@ -200,13 +196,12 @@ Goal::Co LocalDerivationGoal::tryLocalBuild()
/* Are we doing a chroot build? */
{
auto noChroot = parsedDrv->getBoolAttr("__noChroot");
if (settings.sandboxMode == smEnabled) {
if (noChroot)
if (drvOptions->noChroot)
throw Error("derivation '%s' has '__noChroot' set, "
"but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
#if __APPLE__
if (additionalSandboxProfile != "")
if (drvOptions->additionalSandboxProfile != "")
throw Error("derivation '%s' specifies a sandbox profile, "
"but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
#endif
@ -215,7 +210,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
useChroot = derivationType->isSandboxed() && !noChroot;
useChroot = derivationType->isSandboxed() && !drvOptions->noChroot;
}
auto & localStore = getLocalStore();
@ -240,7 +235,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild()
if (useBuildUsers()) {
if (!buildUser)
buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot);
buildUser = acquireUserLock(drvOptions->useUidRange(*drv) ? 65536 : 1, useChroot);
if (!buildUser) {
if (!actLock)
@ -531,13 +526,19 @@ void LocalDerivationGoal::startBuilder()
killSandbox(false);
/* Right platform? */
if (!parsedDrv->canBuildLocally(worker.store))
throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
drv->platform,
concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
worker.store.printStorePath(drvPath),
settings.thisSystem,
concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
if (!drvOptions->canBuildLocally(worker.store, *drv)) {
// since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2
if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") {
throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv->platform);
} else {
throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
drv->platform,
concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)),
worker.store.printStorePath(drvPath),
settings.thisSystem,
concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
}
}
/* Create a temporary directory where the build will take
place. */
@ -622,7 +623,7 @@ void LocalDerivationGoal::startBuilder()
writeStructuredAttrs();
/* Handle exportReferencesGraph(), if set. */
if (!parsedDrv->getStructuredAttrs()) {
if (!parsedDrv->hasStructuredAttrs()) {
/* The `exportReferencesGraph' feature allows the references graph
to be passed to a builder. This attribute should be a list of
pairs [name1 path1 name2 path2 ...]. The references graph of
@ -696,7 +697,7 @@ void LocalDerivationGoal::startBuilder()
PathSet allowedPaths = settings.allowedImpureHostPrefixes;
/* This works like the above, except on a per-derivation level */
auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
auto impurePaths = drvOptions->impureHostDeps;
for (auto & i : impurePaths) {
bool found = false;
@ -716,7 +717,7 @@ void LocalDerivationGoal::startBuilder()
throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
worker.store.printStorePath(drvPath), i);
/* Allow files in __impureHostDeps to be missing; e.g.
/* Allow files in drvOptions->impureHostDeps to be missing; e.g.
macOS 11+ has no /usr/lib/libSystem*.dylib */
pathsInChroot[i] = {i, true};
}
@ -756,10 +757,10 @@ void LocalDerivationGoal::startBuilder()
nobody account. The latter is kind of a hack to support
Samba-in-QEMU. */
createDirs(chrootRootDir + "/etc");
if (parsedDrv->useUidRange())
if (drvOptions->useUidRange(*drv))
chownToBuilder(chrootRootDir + "/etc");
if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536))
if (drvOptions->useUidRange(*drv) && (!buildUser || buildUser->getUIDCount() < 65536))
throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name);
/* Declare the build user's group so that programs get a consistent
@ -800,7 +801,7 @@ void LocalDerivationGoal::startBuilder()
out. */
for (auto & i : drv->outputsAndOptPaths(worker.store)) {
/* If the name isn't known a priori (i.e. floating
content-addressed derivation), the temporary location we use
content-addressing derivation), the temporary location we use
should be fresh. Freshness means it is impossible that the path
is already in the sandbox, so we don't need to worry about
removing it. */
@ -818,7 +819,7 @@ void LocalDerivationGoal::startBuilder()
}
#else
if (parsedDrv->useUidRange())
if (drvOptions->useUidRange(*drv))
throw Error("feature 'uid-range' is not supported on this platform");
#if __APPLE__
/* We don't really have any parent prep work to do (yet?)
@ -828,7 +829,7 @@ void LocalDerivationGoal::startBuilder()
#endif
#endif
} else {
if (parsedDrv->useUidRange())
if (drvOptions->useUidRange(*drv))
throw Error("feature 'uid-range' is only supported in sandboxed builds");
}
@ -873,7 +874,7 @@ void LocalDerivationGoal::startBuilder()
/* Fire up a Nix daemon to process recursive Nix calls from the
builder. */
if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
if (drvOptions->getRequiredSystemFeatures(*drv).count("recursive-nix"))
startDaemon();
/* Run the builder. */
@ -1141,18 +1142,12 @@ void LocalDerivationGoal::initTmpDir()
tmpDirInSandbox = tmpDir;
#endif
/* In non-structured mode, add all bindings specified in the
derivation via the environment, except those listed in the
passAsFile attribute. Those are passed as file names pointing
to temporary files containing the contents. Note that
passAsFile is ignored in structure mode because it's not
needed (attributes are not passed through the environment, so
there is no size constraint). */
if (!parsedDrv->getStructuredAttrs()) {
StringSet passAsFile = tokenizeString<StringSet>(getOr(drv->env, "passAsFile", ""));
/* In non-structured mode, set all bindings either directory in the
environment or via a file, as specified by
`DerivationOptions::passAsFile`. */
if (!parsedDrv->hasStructuredAttrs()) {
for (auto & i : drv->env) {
if (passAsFile.find(i.first) == passAsFile.end()) {
if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) {
env[i.first] = i.second;
} else {
auto hash = hashString(HashAlgorithm::SHA256, i.first);
@ -1229,7 +1224,7 @@ void LocalDerivationGoal::initEnv()
if (!impureEnv.empty())
experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv);
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) {
for (auto & i : drvOptions->impureEnvVars){
auto envVar = impureEnv.find(i);
if (envVar != impureEnv.end()) {
env[i] = envVar->second;
@ -1989,7 +1984,7 @@ void LocalDerivationGoal::runChild()
}
/* Make /etc unwritable */
if (!parsedDrv->useUidRange())
if (!drvOptions->useUidRange(*drv))
chmod_(chrootRootDir + "/etc", 0555);
/* Unshare this mount namespace. This is necessary because
@ -2149,7 +2144,18 @@ void LocalDerivationGoal::runChild()
without file-write* allowed, access() incorrectly returns EPERM
*/
sandboxProfile += "(allow file-read* file-write* process-exec\n";
// We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter.
// See https://github.com/NixOS/nix/issues/4119
// We split our allow groups approximately at half the actual limit, 1 << 16
const int breakpoint = sandboxProfile.length() + (1 << 14);
for (auto & i : pathsInChroot) {
if (sandboxProfile.length() >= breakpoint) {
debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint);
sandboxProfile += ")\n(allow file-read* file-write* process-exec\n";
}
if (i.first != i.second.source)
throw Error(
"can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
@ -2176,7 +2182,7 @@ void LocalDerivationGoal::runChild()
}
sandboxProfile += ")\n";
sandboxProfile += additionalSandboxProfile;
sandboxProfile += drvOptions->additionalSandboxProfile;
} else
sandboxProfile +=
#include "sandbox-minimal.sb"
@ -2185,8 +2191,6 @@ void LocalDerivationGoal::runChild()
debug("Generated sandbox profile:");
debug(sandboxProfile);
bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
/* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */
Path globalTmpDir = canonPath(defaultTempDir(), true);
@ -2199,7 +2203,7 @@ void LocalDerivationGoal::runChild()
Strings sandboxArgs;
sandboxArgs.push_back("_GLOBAL_TMP_DIR");
sandboxArgs.push_back(globalTmpDir);
if (allowLocalNetworking) {
if (drvOptions->allowLocalNetworking) {
sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING");
sandboxArgs.push_back("1");
}
@ -2219,7 +2223,7 @@ void LocalDerivationGoal::runChild()
/* Execute the program. This should not return. */
if (drv->isBuiltin()) {
try {
logger = makeJSONLogger(STDERR_FILENO);
logger = makeJSONLogger(getStandardError());
std::map<std::string, Path> outputs;
for (auto & e : drv->outputs)
@ -2291,7 +2295,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
to do anything here.
We can only early return when the outputs are known a priori. For
floating content-addressed derivations this isn't the case.
floating content-addressing derivations this isn't the case.
*/
if (hook)
return DerivationGoal::registerOutputs();
@ -2389,14 +2393,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
inodesSeen);
bool discardReferences = false;
if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) {
if (auto output = get(*udr, outputName)) {
if (!output->is_boolean())
throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string());
discardReferences = output->get<bool>();
}
}
if (auto udr = get(drvOptions->unsafeDiscardReferences, outputName)) {
discardReferences = *udr;
}
StorePathSet references;
@ -2565,7 +2563,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
case FileIngestionMethod::Git: {
return git::dumpHash(
outputHash.hashAlgo,
{getFSSourceAccessor(), CanonPath(tmpDir + "/tmp")}).hash;
{getFSSourceAccessor(), CanonPath(actualPath)}).hash;
}
}
assert(false);
@ -2873,13 +2871,6 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
auto & outputName = output.first;
auto & info = output.second;
struct Checks
{
bool ignoreSelfRefs = false;
std::optional<uint64_t> maxSize, maxClosureSize;
std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
};
/* Compute the closure and closure size of some output. This
is slightly tricky because some of its references (namely
other outputs) may not be valid yet. */
@ -2911,7 +2902,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
return std::make_pair(std::move(pathsDone), closureSize);
};
auto applyChecks = [&](const Checks & checks)
auto applyChecks = [&](const DerivationOptions::OutputChecks & checks)
{
if (checks.maxSize && info.narSize > *checks.maxSize)
throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
@ -2924,15 +2915,13 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
}
auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive)
{
if (!value) return;
/* Parse a list of reference specifiers. Each element must
either be a store path, or the symbolic name of the output
of the derivation (such as `out'). */
StorePathSet spec;
for (auto & i : *value) {
for (auto & i : value) {
if (worker.store.isStorePath(i))
spec.insert(worker.store.parseStorePath(i));
else if (auto output = get(outputs, i))
@ -2970,73 +2959,35 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
}
};
checkRefs(checks.allowedReferences, true, false);
checkRefs(checks.allowedRequisites, true, true);
checkRefs(checks.disallowedReferences, false, false);
checkRefs(checks.disallowedRequisites, false, true);
/* Mandatory check: absent whitelist, and present but empty
whitelist mean very different things. */
if (auto & refs = checks.allowedReferences) {
checkRefs(*refs, true, false);
}
if (auto & refs = checks.allowedRequisites) {
checkRefs(*refs, true, true);
}
/* Optimization: don't need to do anything when
disallowed and empty set. */
if (!checks.disallowedReferences.empty()) {
checkRefs(checks.disallowedReferences, false, false);
}
if (!checks.disallowedRequisites.empty()) {
checkRefs(checks.disallowedRequisites, false, true);
}
};
if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
if (get(*structuredAttrs, "allowedReferences")){
warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead");
}
if (get(*structuredAttrs, "allowedRequisites")){
warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead");
}
if (get(*structuredAttrs, "disallowedRequisites")){
warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead");
}
if (get(*structuredAttrs, "disallowedReferences")){
warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead");
}
if (get(*structuredAttrs, "maxSize")){
warn("'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead");
}
if (get(*structuredAttrs, "maxClosureSize")){
warn("'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead");
}
if (auto outputChecks = get(*structuredAttrs, "outputChecks")) {
if (auto output = get(*outputChecks, outputName)) {
Checks checks;
std::visit(overloaded{
[&](const DerivationOptions::OutputChecks & checks) {
applyChecks(checks);
},
[&](const std::map<std::string, DerivationOptions::OutputChecks> & checksPerOutput) {
if (auto outputChecks = get(checksPerOutput, outputName))
if (auto maxSize = get(*output, "maxSize"))
checks.maxSize = maxSize->get<uint64_t>();
if (auto maxClosureSize = get(*output, "maxClosureSize"))
checks.maxClosureSize = maxClosureSize->get<uint64_t>();
auto get_ = [&](const std::string & name) -> std::optional<Strings> {
if (auto i = get(*output, name)) {
Strings res;
for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string())
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
res.push_back(j->get<std::string>());
}
checks.disallowedRequisites = res;
return res;
}
return {};
};
checks.allowedReferences = get_("allowedReferences");
checks.allowedRequisites = get_("allowedRequisites");
checks.disallowedReferences = get_("disallowedReferences");
checks.disallowedRequisites = get_("disallowedRequisites");
applyChecks(checks);
}
}
} else {
// legacy non-structured-attributes case
Checks checks;
checks.ignoreSelfRefs = true;
checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
applyChecks(checks);
}
applyChecks(*outputChecks);
},
}, drvOptions->outputChecks);
}
}

7
src/libstore/unix/build/local-derivation-goal.hh
View file

@ -109,11 +109,6 @@ struct LocalDerivationGoal : public DerivationGoal
typedef map<std::string, std::string> Environment;
Environment env;
#if __APPLE__
typedef std::string SandboxProfile;
SandboxProfile additionalSandboxProfile;
#endif
/**
* Hash rewriting.
*/
@ -130,7 +125,7 @@ struct LocalDerivationGoal : public DerivationGoal
* rewrite after the build. Otherwise the regular predetermined paths are
* put here.
*
* - Floating content-addressed derivations do not know their final build
* - Floating content-addressing derivations do not know their final build
* output paths until the outputs are hashed, so random locations are
* used, and then renamed. The randomness helps guard against hidden
* self-references.