nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-30 06:01:00 +01:00
Code Activity

Merge remote-tracking branch 'origin/master' into lazy-trees

Browse source
This commit is contained in:
Eelco Dolstra 2023-01-11 13:26:20 +01:00
commit 29dff7e24e
108 changed files with 2107 additions and 1363 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/libstore/build/derivation-goal.cc
View file

@ -39,7 +39,6 @@
#include <sys/ioctl.h>
#include <net/if.h>
#include <netinet/ip.h>
#include <sys/personality.h>
#include <sys/mman.h>
#include <sched.h>
#include <sys/param.h>
@ -545,7 +544,8 @@ void DerivationGoal::inputsRealised()
However, the impure derivations feature still relies on this
fragile way of doing things, because its builds do not have
a representation in the store, which is a usability problem
in itself */
in itself. When implementing this logic entirely with lookups
make sure that they're cached. */
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
worker.store.computeFSClosure(*outPath, inputPaths);
}

45
src/libstore/build/local-derivation-goal.cc
View file

@ -15,6 +15,7 @@
#include "callback.hh"
#include "json-utils.hh"
#include "cgroup.hh"
#include "personality.hh"
#include <regex>
#include <queue>
@ -24,7 +25,6 @@
#include <termios.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/utsname.h>
#include <sys/resource.h>
#include <sys/socket.h>
@ -37,7 +37,6 @@
#include <sys/ioctl.h>
#include <net/if.h>
#include <netinet/ip.h>
#include <sys/personality.h>
#include <sys/mman.h>
#include <sched.h>
#include <sys/param.h>
@ -1964,33 +1963,7 @@ void LocalDerivationGoal::runChild()
/* Close all other file descriptors. */
closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
#if __linux__
/* Change the personality to 32-bit if we're doing an
i686-linux build on an x86_64-linux machine. */
struct utsname utsbuf;
uname(&utsbuf);
if ((drv->platform == "i686-linux"
&& (settings.thisSystem == "x86_64-linux"
|| (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
|| drv->platform == "armv7l-linux"
|| drv->platform == "armv6l-linux")
{
if (personality(PER_LINUX32) == -1)
throw SysError("cannot set 32-bit personality");
}
/* Impersonate a Linux 2.6 machine to get some determinism in
builds that depend on the kernel version. */
if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
int cur = personality(0xffffffff);
if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
}
/* Disable address space randomization for improved
determinism. */
int cur = personality(0xffffffff);
if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
#endif
setPersonality(drv->platform);
/* Disable core dumps by default. */
struct rlimit limit = { 0, RLIM_INFINITY };
@ -2077,10 +2050,14 @@ void LocalDerivationGoal::runChild()
sandboxProfile += "(deny default (with no-log))\n";
}
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
sandboxProfile +=
#include "sandbox-defaults.sb"
;
if (!derivationType.isSandboxed())
sandboxProfile += "(import \"sandbox-network.sb\")\n";
sandboxProfile +=
#include "sandbox-network.sb"
;
/* Add the output paths we'll use at build-time to the chroot */
sandboxProfile += "(allow file-read* file-write* process-exec\n";
@ -2123,7 +2100,9 @@ void LocalDerivationGoal::runChild()
sandboxProfile += additionalSandboxProfile;
} else
sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
sandboxProfile +=
#include "sandbox-minimal.sb"
;
debug("Generated sandbox profile:");
debug(sandboxProfile);
@ -2148,8 +2127,6 @@ void LocalDerivationGoal::runChild()
args.push_back(sandboxFile);
args.push_back("-D");
args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
args.push_back("-D");
args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
if (allowLocalNetworking) {
args.push_back("-D");
args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1"));

44
src/libstore/build/personality.cc Normal file
View file

@ -0,0 +1,44 @@
#include "personality.hh"
#include "globals.hh"
#if __linux__
#include <sys/utsname.h>
#include <sys/personality.h>
#endif
#include <cstring>
namespace nix {
void setPersonality(std::string_view system)
{
#if __linux__
/* Change the personality to 32-bit if we're doing an
i686-linux build on an x86_64-linux machine. */
struct utsname utsbuf;
uname(&utsbuf);
if ((system == "i686-linux"
&& (std::string_view(SYSTEM) == "x86_64-linux"
|| (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
|| system == "armv7l-linux"
|| system == "armv6l-linux")
{
if (personality(PER_LINUX32) == -1)
throw SysError("cannot set 32-bit personality");
}
/* Impersonate a Linux 2.6 machine to get some determinism in
builds that depend on the kernel version. */
if ((system == "i686-linux" || system == "x86_64-linux") && settings.impersonateLinux26) {
int cur = personality(0xffffffff);
if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
}
/* Disable address space randomization for improved
determinism. */
int cur = personality(0xffffffff);
if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
#endif
}
}

11
src/libstore/build/personality.hh Normal file
View file

@ -0,0 +1,11 @@
#pragma once
#include <string>
namespace nix {
void setPersonality(std::string_view system);
}

4
src/libstore/sandbox-defaults.sb → src/libstore/build/sandbox-defaults.sb
View file

@ -1,3 +1,5 @@
R""(
(define TMPDIR (param "_GLOBAL_TMP_DIR"))
(deny default)
@ -104,3 +106,5 @@
(subpath "/System/Library/Apple/usr/libexec/oah")
(subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist")
(subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist"))
)""

4
src/libstore/sandbox-minimal.sb → src/libstore/build/sandbox-minimal.sb
View file

@ -1,5 +1,9 @@
R""(
(allow default)
; Disallow creating setuid/setgid binaries, since that
; would allow breaking build user isolation.
(deny file-write-setugid)
)""

4
src/libstore/sandbox-network.sb → src/libstore/build/sandbox-network.sb
View file

@ -1,3 +1,5 @@
R""(
; Allow local and remote network traffic.
(allow network* (local ip) (remote ip))
@ -18,3 +20,5 @@
; Allow access to trustd.
(allow mach-lookup (global-name "com.apple.trustd"))
(allow mach-lookup (global-name "com.apple.trustd.agent"))
)""

2
src/libstore/builtins/buildenv.cc
View file

@ -95,7 +95,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
throw Error(
"files '%1%' and '%2%' have the same priority %3%; "
"use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
"or type 'nix profile install --help' if using 'nix profile' to find out how"
"or type 'nix profile install --help' if using 'nix profile' to find out how "
"to change the priority of one of the conflicting packages"
" (0 being the highest priority)",
srcFile, readLink(dstFile), priority);

84
src/libstore/gc.cc
View file

@ -77,60 +77,73 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
}
void LocalStore::addTempRoot(const StorePath & path)
void LocalStore::createTempRootsFile()
{
auto state(_state.lock());
auto fdTempRoots(_fdTempRoots.lock());
/* Create the temporary roots file for this process. */
if (!state->fdTempRoots) {
if (*fdTempRoots) return;
while (1) {
if (pathExists(fnTempRoots))
/* It *must* be stale, since there can be no two
processes with the same pid. */
unlink(fnTempRoots.c_str());
while (1) {
if (pathExists(fnTempRoots))
/* It *must* be stale, since there can be no two
processes with the same pid. */
unlink(fnTempRoots.c_str());
state->fdTempRoots = openLockFile(fnTempRoots, true);
*fdTempRoots = openLockFile(fnTempRoots, true);
debug("acquiring write lock on '%s'", fnTempRoots);
lockFile(state->fdTempRoots.get(), ltWrite, true);
debug("acquiring write lock on '%s'", fnTempRoots);
lockFile(fdTempRoots->get(), ltWrite, true);
/* Check whether the garbage collector didn't get in our
way. */
struct stat st;
if (fstat(state->fdTempRoots.get(), &st) == -1)
throw SysError("statting '%1%'", fnTempRoots);
if (st.st_size == 0) break;
/* Check whether the garbage collector didn't get in our
way. */
struct stat st;
if (fstat(fdTempRoots->get(), &st) == -1)
throw SysError("statting '%1%'", fnTempRoots);
if (st.st_size == 0) break;
/* The garbage collector deleted this file before we could
get a lock. (It won't delete the file after we get a
lock.) Try again. */
}
/* The garbage collector deleted this file before we could get
a lock. (It won't delete the file after we get a lock.)
Try again. */
}
}
void LocalStore::addTempRoot(const StorePath & path)
{
createTempRootsFile();
/* Open/create the global GC lock file. */
{
auto fdGCLock(_fdGCLock.lock());
if (!*fdGCLock)
*fdGCLock = openGCLock();
}
if (!state->fdGCLock)
state->fdGCLock = openGCLock();
restart:
FdLock gcLock(state->fdGCLock.get(), ltRead, false, "");
/* Try to acquire a shared global GC lock (non-blocking). This
only succeeds if the garbage collector is not currently
running. */
FdLock gcLock(_fdGCLock.lock()->get(), ltRead, false, "");
if (!gcLock.acquired) {
/* We couldn't get a shared global GC lock, so the garbage
collector is running. So we have to connect to the garbage
collector and inform it about our root. */
if (!state->fdRootsSocket) {
auto fdRootsSocket(_fdRootsSocket.lock());
if (!*fdRootsSocket) {
auto socketPath = stateDir.get() + gcSocketPath;
debug("connecting to '%s'", socketPath);
state->fdRootsSocket = createUnixDomainSocket();
*fdRootsSocket = createUnixDomainSocket();
try {
nix::connect(state->fdRootsSocket.get(), socketPath);
nix::connect(fdRootsSocket->get(), socketPath);
} catch (SysError & e) {
/* The garbage collector may have exited, so we need to
restart. */
if (e.errNo == ECONNREFUSED) {
debug("GC socket connection refused");
state->fdRootsSocket.close();
fdRootsSocket->close();
goto restart;
}
throw;
@ -139,9 +152,9 @@ void LocalStore::addTempRoot(const StorePath & path)
try {
debug("sending GC root '%s'", printStorePath(path));
writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false);
writeFull(fdRootsSocket->get(), printStorePath(path) + "\n", false);
char c;
readFull(state->fdRootsSocket.get(), &c, 1);
readFull(fdRootsSocket->get(), &c, 1);
assert(c == '1');
debug("got ack for GC root '%s'", printStorePath(path));
} catch (SysError & e) {
@ -149,20 +162,21 @@ void LocalStore::addTempRoot(const StorePath & path)
restart. */
if (e.errNo == EPIPE || e.errNo == ECONNRESET) {
debug("GC socket disconnected");
state->fdRootsSocket.close();
fdRootsSocket->close();
goto restart;
}
throw;
} catch (EndOfFile & e) {
debug("GC socket disconnected");
state->fdRootsSocket.close();
fdRootsSocket->close();
goto restart;
}
}
/* Append the store path to the temporary roots file. */
/* Record the store path in the temporary roots file so it will be
seen by a future run of the garbage collector. */
auto s = printStorePath(path) + '\0';
writeFull(state->fdTempRoots.get(), s);
writeFull(_fdTempRoots.lock()->get(), s);
}

14
src/libstore/globals.cc
View file

@ -291,4 +291,18 @@ void initPlugins()
settings.pluginFiles.pluginsLoaded = true;
}
static bool initLibStoreDone = false;
void assertLibStoreInitialized() {
if (!initLibStoreDone) {
printError("The program must call nix::initNix() before calling any libstore library functions.");
abort();
};
}
void initLibStore() {
initLibStoreDone = true;
}
}

13
src/libstore/globals.hh
View file

@ -329,7 +329,7 @@ public:
Whether to execute builds inside cgroups.
This is only supported on Linux.
Cgroups are required and enabled automatically for derivations
Cgroups are required and enabled automatically for derivations
that require the `uid-range` system feature.
> **Warning**
@ -491,6 +491,9 @@ public:
for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will
only be mounted in the sandbox if it exists in the host filesystem.
If the source is in the Nix store, then its closure will be added to
the sandbox as well.
Depending on how Nix was built, the default value for this option
may be empty or provide `/bin/sh` as a bind-mount of `bash`.
)",
@ -984,4 +987,12 @@ std::vector<Path> getUserConfigFiles();
extern const std::string nixVersion;
/* NB: This is not sufficient. You need to call initNix() */
void initLibStore();
/* It's important to initialize before doing _anything_, which is why we
call upon the programmer to handle this correctly. However, we only add
this in a key locations, so as not to litter the code. */
void assertLibStoreInitialized();
}

8
src/libstore/local-store.cc
View file

@ -91,6 +91,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
if (!lockFile(lockFd.get(), ltWrite, false)) {
printInfo("waiting for exclusive access to the Nix store for ca drvs...");
lockFile(lockFd.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks
lockFile(lockFd.get(), ltWrite, true);
}
@ -299,6 +300,7 @@ LocalStore::LocalStore(const Params & params)
if (!lockFile(globalLock.get(), ltWrite, false)) {
printInfo("waiting for exclusive access to the Nix store...");
lockFile(globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks
lockFile(globalLock.get(), ltWrite, true);
}
@ -439,9 +441,9 @@ LocalStore::~LocalStore()
}
try {
auto state(_state.lock());
if (state->fdTempRoots) {
state->fdTempRoots = -1;
auto fdTempRoots(_fdTempRoots.lock());
if (*fdTempRoots) {
*fdTempRoots = -1;
unlink(fnTempRoots.c_str());
}
} catch (...) {

24
src/libstore/local-store.hh
View file

@ -59,15 +59,6 @@ private:
struct Stmts;
std::unique_ptr<Stmts> stmts;
/* The global GC lock */
AutoCloseFD fdGCLock;
/* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots;
/* Connection to the garbage collector. */
AutoCloseFD fdRootsSocket;
/* The last time we checked whether to do an auto-GC, or an
auto-GC finished. */
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
@ -156,6 +147,21 @@ public:
void addTempRoot(const StorePath & path) override;
private:
void createTempRootsFile();
/* The file to which we write our temporary roots. */
Sync<AutoCloseFD> _fdTempRoots;
/* The global GC lock. */
Sync<AutoCloseFD> _fdGCLock;
/* Connection to the garbage collector. */
Sync<AutoCloseFD> _fdRootsSocket;
public:
void addIndirectRoot(const Path & path) override;
private:

4
src/libstore/local.mk
View file

@ -13,10 +13,6 @@ ifdef HOST_LINUX
libstore_LDFLAGS += -ldl
endif
ifdef HOST_DARWIN
libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
endif
$(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
ifeq ($(ENABLE_S3), 1)

12
src/libstore/lock.cc
View file

@ -123,8 +123,12 @@ struct AutoUserLock : UserLock
std::vector<gid_t> getSupplementaryGIDs() override { return {}; }
static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useChroot)
static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useUserNamespace)
{
#if !defined(__linux__)
useUserNamespace = false;
#endif
settings.requireExperimentalFeature(Xp::AutoAllocateUids);
assert(settings.startId > 0);
assert(settings.uidCount % maxIdsPerBuild == 0);
@ -157,7 +161,7 @@ struct AutoUserLock : UserLock
auto lock = std::make_unique<AutoUserLock>();
lock->fdUserLock = std::move(fd);
lock->firstUid = firstUid;
if (useChroot)
if (useUserNamespace)
lock->firstGid = firstUid;
else {
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
@ -174,10 +178,10 @@ struct AutoUserLock : UserLock
}
};
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot)
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace)
{
if (settings.autoAllocateUids)
return AutoUserLock::acquire(nrIds, useChroot);
return AutoUserLock::acquire(nrIds, useUserNamespace);
else
return SimpleUserLock::acquire();
}

2
src/libstore/lock.hh
View file

@ -31,7 +31,7 @@ struct UserLock
/* Acquire a user lock for a UID range of size `nrIds`. Note that this
may return nullptr if no user is available. */
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot);
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace);
bool useBuildUsers();

42
src/libstore/nar-info-disk-cache.cc
View file

@ -166,16 +166,37 @@ public:
return i->second;
}
std::optional<Cache> queryCacheRaw(State & state, const std::string & uri)
{
auto i = state.caches.find(uri);
if (i == state.caches.end()) {
auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl));
if (!queryCache.next())
return std::nullopt;
state.caches.emplace(uri,
Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
}
return getCache(state, uri);
}
void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
{
retrySQLite<void>([&]() {
auto state(_state.lock());
SQLiteTxn txn(state->db);
// FIXME: race
// To avoid the race, we have to check if maybe someone hasn't yet created
// the cache for this URI in the meantime.
auto cache(queryCacheRaw(*state, uri));
if (cache)
return;
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
txn.commit();
});
}
@ -183,21 +204,12 @@ public:
{
return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
auto state(_state.lock());
auto i = state->caches.find(uri);
if (i == state->caches.end()) {
auto queryCache(state->queryCache.use()(uri)(time(0) - cacheInfoTtl));
if (!queryCache.next())
return std::nullopt;
state->caches.emplace(uri,
Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
}
auto & cache(getCache(*state, uri));
auto cache(queryCacheRaw(*state, uri));
if (!cache)
return std::nullopt;
return CacheInfo {
.wantMassQuery = cache.wantMassQuery,
.priority = cache.priority
.wantMassQuery = cache->wantMassQuery,
.priority = cache->priority
};
});
}

10
src/libstore/realisation.hh
View file

@ -93,4 +93,14 @@ struct RealisedPath {
GENERATE_CMP(RealisedPath, me->raw);
};
class MissingRealisation : public Error
{
public:
MissingRealisation(DrvOutput & outputId)
: Error( "cannot operate on an output of the "
"unbuilt derivation '%s'",
outputId.to_string())
{}
};
}

5
src/libstore/remote-store.cc
View file

@ -879,10 +879,7 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
auto realisation =
queryRealisation(outputId);
if (!realisation)
throw Error(
"cannot operate on an output of unbuilt "
"content-addressed derivation '%s'",
outputId.to_string());
throw MissingRealisation(outputId);
res.builtOutputs.emplace(realisation->id, *realisation);
} else {
// If ca-derivations isn't enabled, assume that

10
src/libstore/sqlite.cc
View file

@ -47,9 +47,13 @@ SQLite::SQLite(const Path & path, bool create)
// `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
// for Linux (WSL) where useSQLiteWAL should be false by default.
const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile";
if (sqlite3_open_v2(path.c_str(), &db,
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK)
throw Error("cannot open SQLite database '%s'", path);
int flags = SQLITE_OPEN_READWRITE;
if (create) flags |= SQLITE_OPEN_CREATE;
int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs);
if (ret != SQLITE_OK) {
const char * err = sqlite3_errstr(ret);
throw Error("cannot open SQLite database '%s': %s", path, err);
}
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
SQLiteError::throw_(db, "setting timeout");

1
src/libstore/store-api.cc
View file

@ -462,6 +462,7 @@ Store::Store(const Params & params)
: StoreConfig(params)
, state({(size_t) pathInfoCacheSize})
{
assertLibStoreInitialized();
}