nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-21 09:49:36 +01:00
Code Activity

Merge remote-tracking branch 'origin/master' into flakes

Browse source
This commit is contained in:
Eelco Dolstra 2019-03-26 11:44:14 +01:00
commit b5565a7081
36 changed files with 470 additions and 180 deletions
Download patch file Download diff file Expand all files Collapse all files

31
src/libstore/download.cc
View file

@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader
writeFull(wakeupPipe.writeSide.get(), " ");
}
#ifdef ENABLE_S3
std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
{
auto [path, params] = splitUriAndParams(uri);
auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
if (slash == std::string::npos)
throw nix::Error("bad S3 URI '%s'", path);
std::string bucketName(path, 5, slash - 5);
std::string key(path, slash + 1);
return {bucketName, key, params};
}
#endif
void enqueueDownload(const DownloadRequest & request,
Callback<DownloadResult> callback) override
{
@ -622,12 +638,15 @@ struct CurlDownloader : public Downloader
// FIXME: do this on a worker thread
try {
#ifdef ENABLE_S3
S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
auto slash = request.uri.find('/', 5);
if (slash == std::string::npos)
throw nix::Error("bad S3 URI '%s'", request.uri);
std::string bucketName(request.uri, 5, slash - 5);
std::string key(request.uri, slash + 1);
auto [bucketName, key, params] = parseS3Uri(request.uri);
std::string profile = get(params, "profile", "");
std::string region = get(params, "region", Aws::Region::US_EAST_1);
std::string scheme = get(params, "scheme", "");
std::string endpoint = get(params, "endpoint", "");
S3Helper s3Helper(profile, region, scheme, endpoint);
// FIXME: implement ETag
auto s3Res = s3Helper.getObject(bucketName, key);
DownloadResult res;

119
src/libstore/gc.cc
View file

@ -129,8 +129,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
check if the root is in a directory in or linked from the
gcroots directory. */
if (settings.checkRootReachability) {
Roots roots = findRoots();
if (roots.find(gcRoot) == roots.end())
Roots roots = findRoots(false);
if (roots[storePath].count(gcRoot) == 0)
printError(
format(
"warning: '%1%' is not in a directory where the garbage collector looks for roots; "
@ -197,10 +197,11 @@ void LocalStore::addTempRoot(const Path & path)
}
std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
{
std::set<std::pair<pid_t, Path>> tempRoots;
static std::string censored = "{censored}";
void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
{
/* Read the `temproots' directory for per-process temporary root
files. */
for (auto & i : readDirectory(tempRootsDir)) {
@ -250,14 +251,12 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds)
Path root(contents, pos, end - pos);
debug("got temporary root '%s'", root);
assertStorePath(root);
tempRoots.emplace(pid, root);
tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid));
pos = end + 1;
}
fds.push_back(fd); /* keep open */
}
return tempRoots;
}
@ -266,7 +265,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
auto foundRoot = [&](const Path & path, const Path & target) {
Path storePath = toStorePath(target);
if (isStorePath(storePath) && isValidPath(storePath))
roots[path] = storePath;
roots[storePath].emplace(path);
else
printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
};
@ -306,7 +305,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
else if (type == DT_REG) {
Path storePath = storeDir + "/" + baseNameOf(path);
if (isStorePath(storePath) && isValidPath(storePath))
roots[path] = storePath;
roots[storePath].emplace(path);
}
}
@ -321,10 +320,8 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
}
Roots LocalStore::findRootsNoTemp()
void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
{
Roots roots;
/* Process direct roots in {gcroots,profiles}. */
findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
@ -333,32 +330,22 @@ Roots LocalStore::findRootsNoTemp()
NIX_ROOT_FINDER environment variable. This is typically used
to add running programs to the set of roots (to prevent them
from being garbage collected). */
size_t n = 0;
for (auto & root : findRuntimeRoots())
roots[fmt("{memory:%d}", n++)] = root;
return roots;
findRuntimeRoots(roots, censor);
}
Roots LocalStore::findRoots()
Roots LocalStore::findRoots(bool censor)
{
Roots roots = findRootsNoTemp();
Roots roots;
findRootsNoTemp(roots, censor);
FDs fds;
pid_t prev = -1;
size_t n = 0;
for (auto & root : readTempRoots(fds)) {
if (prev != root.first) n = 0;
prev = root.first;
roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second;
}
findTempRoots(fds, roots, censor);
return roots;
}
static void readProcLink(const string & file, StringSet & paths)
static void readProcLink(const string & file, Roots & roots)
{
/* 64 is the starting buffer size gnu readlink uses... */
auto bufsiz = ssize_t{64};
@ -377,8 +364,8 @@ try_again:
goto try_again;
}
if (res > 0 && buf[0] == '/')
paths.emplace(static_cast<char *>(buf), res);
return;
roots[std::string(static_cast<char *>(buf), res)]
.emplace(file);
}
static string quoteRegexChars(const string & raw)
@ -387,20 +374,20 @@ static string quoteRegexChars(const string & raw)
return std::regex_replace(raw, specialRegex, R"(\$&)");
}
static void readFileRoots(const char * path, StringSet & paths)
static void readFileRoots(const char * path, Roots & roots)
{
try {
paths.emplace(readFile(path));
roots[readFile(path)].emplace(path);
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != EACCES)
throw;
}
}
PathSet LocalStore::findRuntimeRoots()
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
PathSet roots;
StringSet paths;
Roots unchecked;
auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
struct dirent * ent;
@ -410,10 +397,10 @@ PathSet LocalStore::findRuntimeRoots()
while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt();
if (std::regex_match(ent->d_name, digitsRegex)) {
readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths);
readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths);
readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str();
auto fdStr = fmt("/proc/%s/fd", ent->d_name);
auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
if (!fdDir) {
if (errno == ENOENT || errno == EACCES)
@ -422,9 +409,8 @@ PathSet LocalStore::findRuntimeRoots()
}
struct dirent * fd_ent;
while (errno = 0, fd_ent = readdir(fdDir.get())) {
if (fd_ent->d_name[0] != '.') {
readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths);
}
if (fd_ent->d_name[0] != '.')
readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
}
if (errno) {
if (errno == ESRCH)
@ -434,18 +420,19 @@ PathSet LocalStore::findRuntimeRoots()
fdDir.reset();
try {
auto mapLines =
tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n");
for (const auto& line : mapLines) {
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex))
paths.emplace(match[1]);
unchecked[match[1]].emplace(mapFile);
}
auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true);
auto envFile = fmt("/proc/%s/environ", ent->d_name);
auto envString = readFile(envFile, true);
auto env_end = std::sregex_iterator{};
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
paths.emplace(i->str());
unchecked[i->str()].emplace(envFile);
} catch (SysError & e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
continue;
@ -465,7 +452,7 @@ PathSet LocalStore::findRuntimeRoots()
for (const auto & line : lsofLines) {
std::smatch match;
if (std::regex_match(line, match, lsofRegex))
paths.emplace(match[1]);
unchecked[match[1]].emplace("{lsof}");
}
} catch (ExecError & e) {
/* lsof not installed, lsof failed */
@ -473,21 +460,23 @@ PathSet LocalStore::findRuntimeRoots()
#endif
#if defined(__linux__)
readFileRoots("/proc/sys/kernel/modprobe", paths);
readFileRoots("/proc/sys/kernel/fbsplash", paths);
readFileRoots("/proc/sys/kernel/poweroff_cmd", paths);
readFileRoots("/proc/sys/kernel/modprobe", unchecked);
readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
#endif
for (auto & i : paths)
if (isInStore(i)) {
Path path = toStorePath(i);
if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) {
for (auto & [target, links] : unchecked) {
if (isInStore(target)) {
Path path = toStorePath(target);
if (isStorePath(path) && isValidPath(path)) {
debug(format("got additional root '%1%'") % path);
roots.insert(path);
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
}
}
return roots;
}
}
@ -754,16 +743,20 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
/* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */
printError(format("finding garbage collector roots..."));
Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp();
Roots rootMap;
if (!options.ignoreLiveness)
findRootsNoTemp(rootMap, true);
for (auto & i : rootMap) state.roots.insert(i.second);
for (auto & i : rootMap) state.roots.insert(i.first);
/* Read the temporary roots. This acquires read locks on all
per-process temporary root files. So after this point no paths
can be added to the set of temporary roots. */
FDs fds;
for (auto & root : readTempRoots(fds))
state.tempRoots.insert(root.second);
Roots tempRoots;
findTempRoots(fds, tempRoots, true);
for (auto & root : tempRoots)
state.tempRoots.insert(root.first);
state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
/* After this point the set of roots or temporary roots cannot

8
src/libstore/local-store.hh
View file

@ -180,11 +180,11 @@ private:
typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds);
void findTempRoots(FDs & fds, Roots & roots, bool censor);
public:
Roots findRoots() override;
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;
@ -267,9 +267,9 @@ private:
void findRoots(const Path & path, unsigned char type, Roots & roots);
Roots findRootsNoTemp();
void findRootsNoTemp(Roots & roots, bool censor);
PathSet findRuntimeRoots();
void findRuntimeRoots(Roots & roots, bool censor);
void removeUnusedLinks(const GCState & state);

4
src/libstore/remote-store.cc
View file

@ -596,7 +596,7 @@ void RemoteStore::syncWithGC()
}
Roots RemoteStore::findRoots()
Roots RemoteStore::findRoots(bool censor)
{
auto conn(getConnection());
conn->to << wopFindRoots;
@ -606,7 +606,7 @@ Roots RemoteStore::findRoots()
while (count--) {
Path link = readString(conn->from);
Path target = readStorePath(*this, conn->from);
result[link] = target;
result[target].emplace(link);
}
return result;
}

2
src/libstore/remote-store.hh
View file

@ -82,7 +82,7 @@ public:
void syncWithGC() override;
Roots findRoots() override;
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;

1
src/libstore/s3-binary-cache-store.cc
View file

@ -126,6 +126,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
res->endpointOverride = endpoint;
}
res->requestTimeoutMs = 600 * 1000;
res->connectTimeoutMs = 5 * 1000;
res->retryStrategy = std::make_shared<RetryStrategy>();
res->caFile = settings.caFile;
return res;

16
src/libstore/store-api.cc
View file

@ -842,12 +842,11 @@ namespace nix {
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
{
auto uri(uri_);
Store::Params params(extraParams);
Store::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
@ -873,6 +872,15 @@ ref<Store> openStore(const std::string & uri_,
}
uri = uri_.substr(0, q);
}
return {uri, params};
}
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
{
auto [uri, uriParams] = splitUriAndParams(uri_);
auto params = extraParams;
params.insert(uriParams.begin(), uriParams.end());
for (auto fun : *RegisterStoreImplementation::implementations) {
auto store = fun(uri, params);

14
src/libstore/store-api.hh
View file

@ -11,6 +11,8 @@
#include <atomic>
#include <limits>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <memory>
#include <string>
@ -47,7 +49,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
const uint32_t exportMagic = 0x4558494e;
typedef std::map<Path, Path> Roots;
typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
struct GCOptions
@ -483,8 +485,10 @@ public:
/* Find the roots of the garbage collector. Each root is a pair
(link, storepath) where `link' is the path of the symlink
outside of the Nix store that point to `storePath'. */
virtual Roots findRoots()
outside of the Nix store that point to `storePath'. If
'censor' is true, privacy-sensitive information about roots
found in /proc is censored. */
virtual Roots findRoots(bool censor)
{ unsupported("findRoots"); }
/* Perform a garbage collection. */
@ -798,4 +802,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(bool recursive, const Hash & hash);
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
}