mirror of
https://github.com/NixOS/nix.git
synced 2025-11-21 17:59:36 +01:00
Merge branch 'master' of github.com:NixOS/nix into enum-class
This commit is contained in:
Carlo Nucera
commit
4f597fb901
190 changed files with 7862 additions and 1487 deletions
|
|
@ -327,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
|
|||
}
|
||||
|
||||
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
|
||||
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
// FIXME: some cut&paste from LocalStore::addToStore().
|
||||
|
||||
|
|
@ -336,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
|
|||
small files. */
|
||||
StringSink sink;
|
||||
Hash h;
|
||||
if (recursive) {
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
dumpPath(srcPath, sink, filter);
|
||||
h = hashString(hashAlgo, *sink.s);
|
||||
} else {
|
||||
|
|
@ -345,7 +345,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
|
|||
h = hashString(hashAlgo, s);
|
||||
}
|
||||
|
||||
ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
|
||||
ValidPathInfo info(makeFixedOutputPath(method, h, name));
|
||||
|
||||
addToStore(info, sink.s, repair, CheckSigs, nullptr);
|
||||
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ public:
|
|||
std::shared_ptr<FSAccessor> accessor) override;
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
bool recursive, HashType hashAlgo,
|
||||
FileIngestionMethod method, HashType hashAlgo,
|
||||
PathFilter & filter, RepairFlag repair) override;
|
||||
|
||||
StorePath addTextToStore(const string & name, const string & s,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
#include "affinity.hh"
|
||||
#include "builtins.hh"
|
||||
#include "builtins/buildenv.hh"
|
||||
#include "download.hh"
|
||||
#include "filetransfer.hh"
|
||||
#include "finally.hh"
|
||||
#include "compression.hh"
|
||||
#include "json.hh"
|
||||
|
|
@ -33,7 +33,6 @@
|
|||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/utsname.h>
|
||||
#include <sys/select.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
|
|
@ -43,6 +42,7 @@
|
|||
#include <errno.h>
|
||||
#include <cstring>
|
||||
#include <termios.h>
|
||||
#include <poll.h>
|
||||
|
||||
#include <pwd.h>
|
||||
#include <grp.h>
|
||||
|
|
@ -513,9 +513,10 @@ private:
|
|||
Path fnUserLock;
|
||||
AutoCloseFD fdUserLock;
|
||||
|
||||
bool isEnabled = false;
|
||||
string user;
|
||||
uid_t uid;
|
||||
gid_t gid;
|
||||
uid_t uid = 0;
|
||||
gid_t gid = 0;
|
||||
std::vector<gid_t> supplementaryGIDs;
|
||||
|
||||
public:
|
||||
|
|
@ -528,7 +529,9 @@ public:
|
|||
uid_t getGID() { assert(gid); return gid; }
|
||||
std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
|
||||
|
||||
bool enabled() { return uid != 0; }
|
||||
bool findFreeUser();
|
||||
|
||||
bool enabled() { return isEnabled; }
|
||||
|
||||
};
|
||||
|
||||
|
|
@ -536,6 +539,11 @@ public:
|
|||
UserLock::UserLock()
|
||||
{
|
||||
assert(settings.buildUsersGroup != "");
|
||||
createDirs(settings.nixStateDir + "/userpool");
|
||||
}
|
||||
|
||||
bool UserLock::findFreeUser() {
|
||||
if (enabled()) return true;
|
||||
|
||||
/* Get the members of the build-users-group. */
|
||||
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
|
||||
|
|
@ -565,7 +573,6 @@ UserLock::UserLock()
|
|||
throw Error(format("the user '%1%' in the group '%2%' does not exist")
|
||||
% i % settings.buildUsersGroup);
|
||||
|
||||
createDirs(settings.nixStateDir + "/userpool");
|
||||
|
||||
fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
|
||||
|
||||
|
|
@ -596,16 +603,13 @@ UserLock::UserLock()
|
|||
supplementaryGIDs.resize(ngroups);
|
||||
#endif
|
||||
|
||||
return;
|
||||
isEnabled = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
throw Error(format("all build users are currently in use; "
|
||||
"consider creating additional users and adding them to the '%1%' group")
|
||||
% settings.buildUsersGroup);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
void UserLock::kill()
|
||||
{
|
||||
killUser(uid);
|
||||
|
|
@ -934,6 +938,7 @@ private:
|
|||
void closureRepaired();
|
||||
void inputsRealised();
|
||||
void tryToBuild();
|
||||
void tryLocalBuild();
|
||||
void buildDone();
|
||||
|
||||
/* Is the build hook willing to perform the build? */
|
||||
|
|
@ -1005,6 +1010,8 @@ private:
|
|||
Goal::amDone(result);
|
||||
}
|
||||
|
||||
void started();
|
||||
|
||||
void done(BuildResult::Status status, const string & msg = "");
|
||||
|
||||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
|
|
@ -1392,6 +1399,19 @@ void DerivationGoal::inputsRealised()
|
|||
result = BuildResult();
|
||||
}
|
||||
|
||||
void DerivationGoal::started() {
|
||||
auto msg = fmt(
|
||||
buildMode == bmRepair ? "repairing outputs of '%s'" :
|
||||
buildMode == bmCheck ? "checking outputs of '%s'" :
|
||||
nrRounds > 1 ? "building '%s' (round %d/%d)" :
|
||||
"building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
|
||||
fmt("building '%s'", worker.store.printStorePath(drvPath));
|
||||
if (hook) msg += fmt(" on '%s'", machineName);
|
||||
act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
|
||||
Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
|
||||
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
|
||||
worker.updateProgress();
|
||||
}
|
||||
|
||||
void DerivationGoal::tryToBuild()
|
||||
{
|
||||
|
|
@ -1489,6 +1509,34 @@ void DerivationGoal::tryToBuild()
|
|||
return;
|
||||
}
|
||||
|
||||
state = &DerivationGoal::tryLocalBuild;
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
void DerivationGoal::tryLocalBuild() {
|
||||
|
||||
/* If `build-users-group' is not empty, then we have to build as
|
||||
one of the members of that group. */
|
||||
if (settings.buildUsersGroup != "" && getuid() == 0) {
|
||||
#if defined(__linux__) || defined(__APPLE__)
|
||||
if (!buildUser) buildUser = std::make_unique<UserLock>();
|
||||
|
||||
if (buildUser->findFreeUser()) {
|
||||
/* Make sure that no other processes are executing under this
|
||||
uid. */
|
||||
buildUser->kill();
|
||||
} else {
|
||||
debug("waiting for build users");
|
||||
worker.waitForAWhile(shared_from_this());
|
||||
return;
|
||||
}
|
||||
#else
|
||||
/* Don't know how to block the creation of setuid/setgid
|
||||
binaries on this platform. */
|
||||
throw Error("build users are not supported on this platform for security reasons");
|
||||
#endif
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
/* Okay, we have to build. */
|
||||
|
|
@ -1686,6 +1734,7 @@ void DerivationGoal::buildDone()
|
|||
}
|
||||
|
||||
if (buildMode == bmCheck) {
|
||||
deleteTmpDir(true);
|
||||
done(BuildResult::Built);
|
||||
return;
|
||||
}
|
||||
|
|
@ -1948,22 +1997,6 @@ void DerivationGoal::startBuilder()
|
|||
#endif
|
||||
}
|
||||
|
||||
/* If `build-users-group' is not empty, then we have to build as
|
||||
one of the members of that group. */
|
||||
if (settings.buildUsersGroup != "" && getuid() == 0) {
|
||||
#if defined(__linux__) || defined(__APPLE__)
|
||||
buildUser = std::make_unique<UserLock>();
|
||||
|
||||
/* Make sure that no other processes are executing under this
|
||||
uid. */
|
||||
buildUser->kill();
|
||||
#else
|
||||
/* Don't know how to block the creation of setuid/setgid
|
||||
binaries on this platform. */
|
||||
throw Error("build users are not supported on this platform for security reasons");
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Create a temporary directory where the build will take
|
||||
place. */
|
||||
tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
|
||||
|
|
@ -2167,7 +2200,7 @@ void DerivationGoal::startBuilder()
|
|||
if (needsHashRewrite()) {
|
||||
|
||||
if (pathExists(homeDir))
|
||||
throw Error(format("directory '%1%' exists; please remove it") % homeDir);
|
||||
throw Error(format("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing") % homeDir);
|
||||
|
||||
/* We're not doing a chroot build, but we have some valid
|
||||
output paths. Since we can't just overwrite or delete
|
||||
|
|
@ -2255,10 +2288,13 @@ void DerivationGoal::startBuilder()
|
|||
|
||||
if (chown(slaveName.c_str(), buildUser->getUID(), 0))
|
||||
throw SysError("changing owner of pseudoterminal slave");
|
||||
} else {
|
||||
}
|
||||
#if __APPLE__
|
||||
else {
|
||||
if (grantpt(builderOut.readSide.get()))
|
||||
throw SysError("granting access to pseudoterminal slave");
|
||||
}
|
||||
#endif
|
||||
|
||||
#if 0
|
||||
// Mount the pt in the sandbox so that the "tty" command works.
|
||||
|
|
@ -2471,7 +2507,7 @@ void DerivationGoal::initTmpDir() {
|
|||
auto hash = hashString(HashType::SHA256, i.first);
|
||||
string fn = ".attr-" + hash.to_string(Base::Base32, false);
|
||||
Path p = tmpDir + "/" + fn;
|
||||
writeFile(p, i.second);
|
||||
writeFile(p, rewriteStrings(i.second, inputRewrites));
|
||||
chownToBuilder(p);
|
||||
env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
|
||||
}
|
||||
|
|
@ -2718,7 +2754,7 @@ struct RestrictedStore : public LocalFSStore
|
|||
{ throw Error("queryPathFromHashPart"); }
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
bool recursive = true, HashType hashAlgo = HashType::SHA256,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256,
|
||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
|
||||
{ throw Error("addToStore"); }
|
||||
|
||||
|
|
@ -2731,9 +2767,9 @@ struct RestrictedStore : public LocalFSStore
|
|||
}
|
||||
|
||||
StorePath addToStoreFromDump(const string & dump, const string & name,
|
||||
bool recursive = true, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) override
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) override
|
||||
{
|
||||
auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair);
|
||||
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
|
||||
goal.addDependency(path);
|
||||
return path;
|
||||
}
|
||||
|
|
@ -3157,7 +3193,7 @@ void DerivationGoal::runChild()
|
|||
// Only use nss functions to resolve hosts and
|
||||
// services. Don’t use it for anything else that may
|
||||
// be configured for this system. This limits the
|
||||
// potential impurities introduced in fixed outputs.
|
||||
// potential impurities introduced in fixed-outputs.
|
||||
writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
|
||||
|
||||
ss.push_back("/etc/services");
|
||||
|
|
@ -3542,6 +3578,29 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
|
|||
}
|
||||
|
||||
|
||||
static void moveCheckToStore(const Path & src, const Path & dst)
|
||||
{
|
||||
/* For the rename of directory to succeed, we must be running as root or
|
||||
the directory must be made temporarily writable (to update the
|
||||
directory's parent link ".."). */
|
||||
struct stat st;
|
||||
if (lstat(src.c_str(), &st) == -1) {
|
||||
throw SysError(format("getting attributes of path '%1%'") % src);
|
||||
}
|
||||
|
||||
bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
|
||||
|
||||
if (changePerm)
|
||||
chmod_(src, st.st_mode | S_IWUSR);
|
||||
|
||||
if (rename(src.c_str(), dst.c_str()))
|
||||
throw SysError(format("renaming '%1%' to '%2%'") % src % dst);
|
||||
|
||||
if (changePerm)
|
||||
chmod_(dst, st.st_mode);
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::registerOutputs()
|
||||
{
|
||||
/* When using a build hook, the build hook can register the output
|
||||
|
|
@ -3653,21 +3712,24 @@ void DerivationGoal::registerOutputs()
|
|||
|
||||
if (fixedOutput) {
|
||||
|
||||
bool recursive; Hash h;
|
||||
i.second.parseHashInfo(recursive, h);
|
||||
FileIngestionMethod outputHashMode; Hash h;
|
||||
i.second.parseHashInfo(outputHashMode, h);
|
||||
|
||||
if (!recursive) {
|
||||
if (outputHashMode == FileIngestionMethod::Flat) {
|
||||
/* The output path should be a regular file without execute permission. */
|
||||
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
|
||||
throw BuildError(
|
||||
format("output path '%1%' should be a non-executable regular file") % path);
|
||||
format("output path '%1%' should be a non-executable regular file "
|
||||
"since recursive hashing is not enabled (outputHashMode=flat)") % path);
|
||||
}
|
||||
|
||||
/* Check the hash. In hash mode, move the path produced by
|
||||
the derivation to its content-addressed location. */
|
||||
Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
|
||||
Hash h2 = outputHashMode == FileIngestionMethod::Recursive
|
||||
? hashPath(h.type, actualPath).first
|
||||
: hashFile(h.type, actualPath);
|
||||
|
||||
auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name());
|
||||
auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name());
|
||||
|
||||
if (h != h2) {
|
||||
|
||||
|
|
@ -3696,7 +3758,7 @@ void DerivationGoal::registerOutputs()
|
|||
else
|
||||
assert(worker.store.parseStorePath(path) == dest);
|
||||
|
||||
ca = makeFixedOutputCA(recursive, h2);
|
||||
ca = makeFixedOutputCA(outputHashMode, h2);
|
||||
}
|
||||
|
||||
/* Get rid of all weird permissions. This also checks that
|
||||
|
|
@ -3720,8 +3782,7 @@ void DerivationGoal::registerOutputs()
|
|||
if (settings.runDiffHook || settings.keepFailed) {
|
||||
Path dst = worker.store.toRealPath(path + checkSuffix);
|
||||
deletePath(dst);
|
||||
if (rename(actualPath.c_str(), dst.c_str()))
|
||||
throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
|
||||
moveCheckToStore(actualPath, dst);
|
||||
|
||||
handleDiffHook(
|
||||
buildUser ? buildUser->getUID() : getuid(),
|
||||
|
|
@ -3729,10 +3790,10 @@ void DerivationGoal::registerOutputs()
|
|||
path, dst, worker.store.printStorePath(drvPath), tmpDir);
|
||||
|
||||
throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
|
||||
worker.store.printStorePath(drvPath), path, dst);
|
||||
worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst);
|
||||
} else
|
||||
throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
|
||||
worker.store.printStorePath(drvPath), path);
|
||||
worker.store.printStorePath(drvPath), worker.store.toRealPath(path));
|
||||
}
|
||||
|
||||
/* Since we verified the build, it's now ultimately trusted. */
|
||||
|
|
@ -3918,7 +3979,9 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
|
|||
|
||||
auto spec = parseReferenceSpecifiers(worker.store, *drv, *value);
|
||||
|
||||
auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references);
|
||||
auto used = recursive
|
||||
? cloneStorePathSet(getClosure(info.path).first)
|
||||
: cloneStorePathSet(info.references);
|
||||
|
||||
if (recursive && checks.ignoreSelfRefs)
|
||||
used.erase(info.path);
|
||||
|
|
@ -4772,8 +4835,7 @@ void Worker::waitForInput()
|
|||
terminated. */
|
||||
|
||||
bool useTimeout = false;
|
||||
struct timeval timeout;
|
||||
timeout.tv_usec = 0;
|
||||
long timeout = 0;
|
||||
auto before = steady_time_point::clock::now();
|
||||
|
||||
/* If we're monitoring for silence on stdout/stderr, or if there
|
||||
|
|
@ -4791,7 +4853,7 @@ void Worker::waitForInput()
|
|||
nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
|
||||
}
|
||||
if (nearest != steady_time_point::max()) {
|
||||
timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
|
||||
timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
|
||||
useTimeout = true;
|
||||
}
|
||||
|
||||
|
|
@ -4800,32 +4862,30 @@ void Worker::waitForInput()
|
|||
if (!waitingForAWhile.empty()) {
|
||||
useTimeout = true;
|
||||
if (lastWokenUp == steady_time_point::min())
|
||||
printError("waiting for locks or build slots...");
|
||||
printError("waiting for locks, build slots or build users...");
|
||||
if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
|
||||
timeout.tv_sec = std::max(1L,
|
||||
timeout = std::max(1L,
|
||||
(long) std::chrono::duration_cast<std::chrono::seconds>(
|
||||
lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
|
||||
} else lastWokenUp = steady_time_point::min();
|
||||
|
||||
if (useTimeout)
|
||||
vomit("sleeping %d seconds", timeout.tv_sec);
|
||||
vomit("sleeping %d seconds", timeout);
|
||||
|
||||
/* Use select() to wait for the input side of any logger pipe to
|
||||
become `available'. Note that `available' (i.e., non-blocking)
|
||||
includes EOF. */
|
||||
fd_set fds;
|
||||
FD_ZERO(&fds);
|
||||
int fdMax = 0;
|
||||
std::vector<struct pollfd> pollStatus;
|
||||
std::map <int, int> fdToPollStatus;
|
||||
for (auto & i : children) {
|
||||
for (auto & j : i.fds) {
|
||||
if (j >= FD_SETSIZE)
|
||||
throw Error("reached FD_SETSIZE limit");
|
||||
FD_SET(j, &fds);
|
||||
if (j >= fdMax) fdMax = j + 1;
|
||||
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
|
||||
fdToPollStatus[j] = pollStatus.size() - 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) {
|
||||
if (poll(pollStatus.data(), pollStatus.size(),
|
||||
useTimeout ? timeout * 1000 : -1) == -1) {
|
||||
if (errno == EINTR) return;
|
||||
throw SysError("waiting for input");
|
||||
}
|
||||
|
|
@ -4846,7 +4906,7 @@ void Worker::waitForInput()
|
|||
set<int> fds2(j->fds);
|
||||
std::vector<unsigned char> buffer(4096);
|
||||
for (auto & k : fds2) {
|
||||
if (FD_ISSET(k, &fds)) {
|
||||
if (pollStatus.at(fdToPollStatus.at(k)).revents) {
|
||||
ssize_t rd = read(k, buffer.data(), buffer.size());
|
||||
// FIXME: is there a cleaner way to handle pt close
|
||||
// than EIO? Is this even standard?
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#include "builtins.hh"
|
||||
#include "download.hh"
|
||||
#include "filetransfer.hh"
|
||||
#include "store-api.hh"
|
||||
#include "archive.hh"
|
||||
#include "compression.hh"
|
||||
|
|
@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
|||
auto mainUrl = getAttr("url");
|
||||
bool unpack = get(drv.env, "unpack").value_or("") == "1";
|
||||
|
||||
/* Note: have to use a fresh downloader here because we're in
|
||||
/* Note: have to use a fresh fileTransfer here because we're in
|
||||
a forked process. */
|
||||
auto downloader = makeDownloader();
|
||||
auto fileTransfer = makeFileTransfer();
|
||||
|
||||
auto fetch = [&](const std::string & url) {
|
||||
|
||||
|
|
@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
|||
|
||||
/* No need to do TLS verification, because we check the hash of
|
||||
the result anyway. */
|
||||
DownloadRequest request(url);
|
||||
FileTransferRequest request(url);
|
||||
request.verifyTLS = false;
|
||||
request.decompress = false;
|
||||
|
||||
auto decompressor = makeDecompressionSink(
|
||||
unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
|
||||
downloader->download(std::move(request), *decompressor);
|
||||
fileTransfer->download(std::move(request), *decompressor);
|
||||
decompressor->finish();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -364,20 +364,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
}
|
||||
|
||||
case wopAddToStore: {
|
||||
bool fixed, recursive;
|
||||
std::string s, baseName;
|
||||
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
|
||||
/* Compatibility hack. */
|
||||
if (!fixed) {
|
||||
s = "sha256";
|
||||
recursive = true;
|
||||
FileIngestionMethod method;
|
||||
{
|
||||
bool fixed, recursive;
|
||||
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
|
||||
method = FileIngestionMethod { recursive };
|
||||
/* Compatibility hack. */
|
||||
if (!fixed) {
|
||||
s = "sha256";
|
||||
method = FileIngestionMethod::Recursive;
|
||||
}
|
||||
}
|
||||
HashType hashAlgo = parseHashType(s);
|
||||
|
||||
TeeSource savedNAR(from);
|
||||
RetrieveRegularNARSink savedRegular;
|
||||
|
||||
if (recursive) {
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
/* Get the entire NAR dump from the client and save it to
|
||||
a string so that we can pass it to
|
||||
addToStoreFromDump(). */
|
||||
|
|
@ -389,7 +393,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
logger->startWork();
|
||||
if (!savedRegular.regular) throw Error("regular file expected");
|
||||
|
||||
auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
|
||||
auto path = store->addToStoreFromDump(
|
||||
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
|
||||
baseName,
|
||||
method,
|
||||
hashAlgo);
|
||||
logger->stopWork();
|
||||
|
||||
to << store->printStorePath(path);
|
||||
|
|
|
|||
|
|
@ -9,13 +9,13 @@
|
|||
namespace nix {
|
||||
|
||||
|
||||
void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
|
||||
void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
|
||||
{
|
||||
recursive = false;
|
||||
recursive = FileIngestionMethod::Flat;
|
||||
string algo = hashAlgo;
|
||||
|
||||
if (string(algo, 0, 2) == "r:") {
|
||||
recursive = true;
|
||||
recursive = FileIngestionMethod::Recursive;
|
||||
algo = string(algo, 2);
|
||||
}
|
||||
|
||||
|
|
@ -378,7 +378,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
|
|||
if (h == drvHashes.end()) {
|
||||
assert(store.isValidPath(i.first));
|
||||
h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
|
||||
readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first;
|
||||
readDerivation(store, store.toRealPath(i.first)), false)).first;
|
||||
}
|
||||
inputs2.insert_or_assign(h->second.to_string(Base::Base16, false), i.second);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ struct DerivationOutput
|
|||
, hashAlgo(std::move(hashAlgo))
|
||||
, hash(std::move(hash))
|
||||
{ }
|
||||
void parseHashInfo(bool & recursive, Hash & hash) const;
|
||||
void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
|
||||
};
|
||||
|
||||
typedef std::map<string, DerivationOutput> DerivationOutputs;
|
||||
|
|
|
|||
|
|
@ -1,14 +1,10 @@
|
|||
#include "download.hh"
|
||||
#include "filetransfer.hh"
|
||||
#include "util.hh"
|
||||
#include "globals.hh"
|
||||
#include "hash.hh"
|
||||
#include "store-api.hh"
|
||||
#include "archive.hh"
|
||||
#include "s3.hh"
|
||||
#include "compression.hh"
|
||||
#include "pathlocks.hh"
|
||||
#include "finally.hh"
|
||||
#include "tarfile.hh"
|
||||
|
||||
#ifdef ENABLE_S3
|
||||
#include <aws/core/client/ClientConfiguration.h>
|
||||
|
|
@ -31,13 +27,9 @@ using namespace std::string_literals;
|
|||
|
||||
namespace nix {
|
||||
|
||||
DownloadSettings downloadSettings;
|
||||
FileTransferSettings fileTransferSettings;
|
||||
|
||||
static GlobalConfig::Register r1(&downloadSettings);
|
||||
|
||||
CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
|
||||
: uri(uri), ttl(settings.tarballTtl)
|
||||
{ }
|
||||
static GlobalConfig::Register r1(&fileTransferSettings);
|
||||
|
||||
std::string resolveUri(const std::string & uri)
|
||||
{
|
||||
|
|
@ -47,21 +39,21 @@ std::string resolveUri(const std::string & uri)
|
|||
return uri;
|
||||
}
|
||||
|
||||
struct CurlDownloader : public Downloader
|
||||
struct curlFileTransfer : public FileTransfer
|
||||
{
|
||||
CURLM * curlm = 0;
|
||||
|
||||
std::random_device rd;
|
||||
std::mt19937 mt19937;
|
||||
|
||||
struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
|
||||
struct TransferItem : public std::enable_shared_from_this<TransferItem>
|
||||
{
|
||||
CurlDownloader & downloader;
|
||||
DownloadRequest request;
|
||||
DownloadResult result;
|
||||
curlFileTransfer & fileTransfer;
|
||||
FileTransferRequest request;
|
||||
FileTransferResult result;
|
||||
Activity act;
|
||||
bool done = false; // whether either the success or failure function has been called
|
||||
Callback<DownloadResult> callback;
|
||||
Callback<FileTransferResult> callback;
|
||||
CURL * req = 0;
|
||||
bool active = false; // whether the handle has been added to the multi object
|
||||
std::string status;
|
||||
|
|
@ -80,10 +72,10 @@ struct CurlDownloader : public Downloader
|
|||
|
||||
curl_off_t writtenToSink = 0;
|
||||
|
||||
DownloadItem(CurlDownloader & downloader,
|
||||
const DownloadRequest & request,
|
||||
Callback<DownloadResult> && callback)
|
||||
: downloader(downloader)
|
||||
TransferItem(curlFileTransfer & fileTransfer,
|
||||
const FileTransferRequest & request,
|
||||
Callback<FileTransferResult> && callback)
|
||||
: fileTransfer(fileTransfer)
|
||||
, request(request)
|
||||
, act(*logger, Verbosity::Talkative, ActivityType::Download,
|
||||
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
|
||||
|
|
@ -91,8 +83,15 @@ struct CurlDownloader : public Downloader
|
|||
, callback(std::move(callback))
|
||||
, finalSink([this](const unsigned char * data, size_t len) {
|
||||
if (this->request.dataCallback) {
|
||||
writtenToSink += len;
|
||||
this->request.dataCallback((char *) data, len);
|
||||
long httpStatus = 0;
|
||||
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
|
||||
|
||||
/* Only write data to the sink if this is a
|
||||
successful response. */
|
||||
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
|
||||
writtenToSink += len;
|
||||
this->request.dataCallback((char *) data, len);
|
||||
}
|
||||
} else
|
||||
this->result.data->append((char *) data, len);
|
||||
})
|
||||
|
|
@ -103,17 +102,17 @@ struct CurlDownloader : public Downloader
|
|||
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
|
||||
}
|
||||
|
||||
~DownloadItem()
|
||||
~TransferItem()
|
||||
{
|
||||
if (req) {
|
||||
if (active)
|
||||
curl_multi_remove_handle(downloader.curlm, req);
|
||||
curl_multi_remove_handle(fileTransfer.curlm, req);
|
||||
curl_easy_cleanup(req);
|
||||
}
|
||||
if (requestHeaders) curl_slist_free_all(requestHeaders);
|
||||
try {
|
||||
if (!done)
|
||||
fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
|
||||
fail(FileTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri));
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
|
|
@ -157,7 +156,7 @@ struct CurlDownloader : public Downloader
|
|||
|
||||
static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
|
||||
{
|
||||
return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb);
|
||||
return ((TransferItem *) userp)->writeCallback(contents, size, nmemb);
|
||||
}
|
||||
|
||||
size_t headerCallback(void * contents, size_t size, size_t nmemb)
|
||||
|
|
@ -199,7 +198,7 @@ struct CurlDownloader : public Downloader
|
|||
|
||||
static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
|
||||
{
|
||||
return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb);
|
||||
return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
|
||||
}
|
||||
|
||||
int progressCallback(double dltotal, double dlnow)
|
||||
|
|
@ -214,7 +213,7 @@ struct CurlDownloader : public Downloader
|
|||
|
||||
static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
|
||||
{
|
||||
return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow);
|
||||
return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
|
||||
}
|
||||
|
||||
static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
|
||||
|
|
@ -238,7 +237,7 @@ struct CurlDownloader : public Downloader
|
|||
|
||||
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
|
||||
{
|
||||
return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
|
||||
return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
|
||||
}
|
||||
|
||||
void init()
|
||||
|
|
@ -249,7 +248,7 @@ struct CurlDownloader : public Downloader
|
|||
|
||||
if (verbosity >= Verbosity::Vomit) {
|
||||
curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
|
||||
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback);
|
||||
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback);
|
||||
}
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
|
||||
|
|
@ -258,19 +257,19 @@ struct CurlDownloader : public Downloader
|
|||
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
|
||||
curl_easy_setopt(req, CURLOPT_USERAGENT,
|
||||
("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
|
||||
(downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
|
||||
(fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str());
|
||||
#if LIBCURL_VERSION_NUM >= 0x072b00
|
||||
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
|
||||
#endif
|
||||
#if LIBCURL_VERSION_NUM >= 0x072f00
|
||||
if (downloadSettings.enableHttp2)
|
||||
if (fileTransferSettings.enableHttp2)
|
||||
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
|
||||
else
|
||||
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
|
||||
#endif
|
||||
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
|
||||
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
|
||||
curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
|
||||
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper);
|
||||
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
|
||||
curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
|
||||
|
|
@ -298,10 +297,10 @@ struct CurlDownloader : public Downloader
|
|||
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
|
||||
}
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
|
||||
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
|
||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
|
||||
|
||||
/* If no file exist in the specified path, curl continues to work
|
||||
anyway as if netrc support was disabled. */
|
||||
|
|
@ -390,6 +389,7 @@ struct CurlDownloader : public Downloader
|
|||
case CURLE_SSL_CACERT_BADFILE:
|
||||
case CURLE_TOO_MANY_REDIRECTS:
|
||||
case CURLE_WRITE_ERROR:
|
||||
case CURLE_UNSUPPORTED_PROTOCOL:
|
||||
err = Misc;
|
||||
break;
|
||||
default: // Shut up warnings
|
||||
|
|
@ -401,14 +401,14 @@ struct CurlDownloader : public Downloader
|
|||
|
||||
auto exc =
|
||||
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
|
||||
? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
|
||||
? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
|
||||
: httpStatus != 0
|
||||
? DownloadError(err,
|
||||
? FileTransferError(err,
|
||||
fmt("unable to %s '%s': HTTP error %d",
|
||||
request.verb(), request.uri, httpStatus)
|
||||
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
|
||||
)
|
||||
: DownloadError(err,
|
||||
: FileTransferError(err,
|
||||
fmt("unable to %s '%s': %s (%d)",
|
||||
request.verb(), request.uri, curl_easy_strerror(code), code));
|
||||
|
||||
|
|
@ -422,13 +422,13 @@ struct CurlDownloader : public Downloader
|
|||
|| writtenToSink == 0
|
||||
|| (acceptRanges && encoding.empty())))
|
||||
{
|
||||
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
|
||||
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
|
||||
if (writtenToSink)
|
||||
warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
|
||||
else
|
||||
warn("%s; retrying in %d ms", exc.what(), ms);
|
||||
embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
|
||||
downloader.enqueueItem(shared_from_this());
|
||||
fileTransfer.enqueueItem(shared_from_this());
|
||||
}
|
||||
else
|
||||
fail(exc);
|
||||
|
|
@ -439,12 +439,12 @@ struct CurlDownloader : public Downloader
|
|||
struct State
|
||||
{
|
||||
struct EmbargoComparator {
|
||||
bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
|
||||
bool operator() (const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2) {
|
||||
return i1->embargo > i2->embargo;
|
||||
}
|
||||
};
|
||||
bool quit = false;
|
||||
std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
|
||||
std::priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator> incoming;
|
||||
};
|
||||
|
||||
Sync<State> state_;
|
||||
|
|
@ -456,7 +456,7 @@ struct CurlDownloader : public Downloader
|
|||
|
||||
std::thread workerThread;
|
||||
|
||||
CurlDownloader()
|
||||
curlFileTransfer()
|
||||
: mt19937(rd())
|
||||
{
|
||||
static std::once_flag globalInit;
|
||||
|
|
@ -469,7 +469,7 @@ struct CurlDownloader : public Downloader
|
|||
#endif
|
||||
#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
|
||||
curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
|
||||
downloadSettings.httpConnections.get());
|
||||
fileTransferSettings.httpConnections.get());
|
||||
#endif
|
||||
|
||||
wakeupPipe.create();
|
||||
|
|
@ -478,7 +478,7 @@ struct CurlDownloader : public Downloader
|
|||
workerThread = std::thread([&]() { workerThreadEntry(); });
|
||||
}
|
||||
|
||||
~CurlDownloader()
|
||||
~curlFileTransfer()
|
||||
{
|
||||
stopWorkerThread();
|
||||
|
||||
|
|
@ -504,7 +504,7 @@ struct CurlDownloader : public Downloader
|
|||
stopWorkerThread();
|
||||
});
|
||||
|
||||
std::map<CURL *, std::shared_ptr<DownloadItem>> items;
|
||||
std::map<CURL *, std::shared_ptr<TransferItem>> items;
|
||||
|
||||
bool quit = false;
|
||||
|
||||
|
|
@ -561,7 +561,7 @@ struct CurlDownloader : public Downloader
|
|||
throw SysError("reading curl wakeup socket");
|
||||
}
|
||||
|
||||
std::vector<std::shared_ptr<DownloadItem>> incoming;
|
||||
std::vector<std::shared_ptr<TransferItem>> incoming;
|
||||
auto now = std::chrono::steady_clock::now();
|
||||
|
||||
{
|
||||
|
|
@ -609,7 +609,7 @@ struct CurlDownloader : public Downloader
|
|||
}
|
||||
}
|
||||
|
||||
void enqueueItem(std::shared_ptr<DownloadItem> item)
|
||||
void enqueueItem(std::shared_ptr<TransferItem> item)
|
||||
{
|
||||
if (item->request.data
|
||||
&& !hasPrefix(item->request.uri, "http://")
|
||||
|
|
@ -641,8 +641,8 @@ struct CurlDownloader : public Downloader
|
|||
}
|
||||
#endif
|
||||
|
||||
void enqueueDownload(const DownloadRequest & request,
|
||||
Callback<DownloadResult> callback) override
|
||||
void enqueueFileTransfer(const FileTransferRequest & request,
|
||||
Callback<FileTransferResult> callback) override
|
||||
{
|
||||
/* Ugly hack to support s3:// URIs. */
|
||||
if (hasPrefix(request.uri, "s3://")) {
|
||||
|
|
@ -660,9 +660,9 @@ struct CurlDownloader : public Downloader
|
|||
|
||||
// FIXME: implement ETag
|
||||
auto s3Res = s3Helper.getObject(bucketName, key);
|
||||
DownloadResult res;
|
||||
FileTransferResult res;
|
||||
if (!s3Res.data)
|
||||
throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
|
||||
throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
|
||||
res.data = s3Res.data;
|
||||
callback(std::move(res));
|
||||
#else
|
||||
|
|
@ -672,26 +672,26 @@ struct CurlDownloader : public Downloader
|
|||
return;
|
||||
}
|
||||
|
||||
enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
|
||||
enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
|
||||
}
|
||||
};
|
||||
|
||||
ref<Downloader> getDownloader()
|
||||
ref<FileTransfer> getFileTransfer()
|
||||
{
|
||||
static ref<Downloader> downloader = makeDownloader();
|
||||
return downloader;
|
||||
static ref<FileTransfer> fileTransfer = makeFileTransfer();
|
||||
return fileTransfer;
|
||||
}
|
||||
|
||||
ref<Downloader> makeDownloader()
|
||||
ref<FileTransfer> makeFileTransfer()
|
||||
{
|
||||
return make_ref<CurlDownloader>();
|
||||
return make_ref<curlFileTransfer>();
|
||||
}
|
||||
|
||||
std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request)
|
||||
std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
|
||||
{
|
||||
auto promise = std::make_shared<std::promise<DownloadResult>>();
|
||||
enqueueDownload(request,
|
||||
{[promise](std::future<DownloadResult> fut) {
|
||||
auto promise = std::make_shared<std::promise<FileTransferResult>>();
|
||||
enqueueFileTransfer(request,
|
||||
{[promise](std::future<FileTransferResult> fut) {
|
||||
try {
|
||||
promise->set_value(fut.get());
|
||||
} catch (...) {
|
||||
|
|
@ -701,15 +701,21 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
|
|||
return promise->get_future();
|
||||
}
|
||||
|
||||
DownloadResult Downloader::download(const DownloadRequest & request)
|
||||
FileTransferResult FileTransfer::download(const FileTransferRequest & request)
|
||||
{
|
||||
return enqueueDownload(request).get();
|
||||
return enqueueFileTransfer(request).get();
|
||||
}
|
||||
|
||||
void Downloader::download(DownloadRequest && request, Sink & sink)
|
||||
FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
|
||||
{
|
||||
/* Note: this method is the same as download, but helps in readability */
|
||||
return enqueueFileTransfer(request).get();
|
||||
}
|
||||
|
||||
void FileTransfer::download(FileTransferRequest && request, Sink & sink)
|
||||
{
|
||||
/* Note: we can't call 'sink' via request.dataCallback, because
|
||||
that would cause the sink to execute on the downloader
|
||||
that would cause the sink to execute on the fileTransfer
|
||||
thread. If 'sink' is a coroutine, this will fail. Also, if the
|
||||
sink is expensive (e.g. one that does decompression and writing
|
||||
to the Nix store), it would stall the download thread too much.
|
||||
|
|
@ -755,8 +761,8 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
|
|||
state->avail.notify_one();
|
||||
};
|
||||
|
||||
enqueueDownload(request,
|
||||
{[_state](std::future<DownloadResult> fut) {
|
||||
enqueueFileTransfer(request,
|
||||
{[_state](std::future<FileTransferResult> fut) {
|
||||
auto state(_state->lock());
|
||||
state->quit = true;
|
||||
try {
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
struct DownloadSettings : Config
|
||||
struct FileTransferSettings : Config
|
||||
{
|
||||
Setting<bool> enableHttp2{this, true, "http2",
|
||||
"Whether to enable HTTP/2 support."};
|
||||
|
|
@ -31,15 +31,15 @@ struct DownloadSettings : Config
|
|||
"How often Nix will attempt to download a file before giving up."};
|
||||
};
|
||||
|
||||
extern DownloadSettings downloadSettings;
|
||||
extern FileTransferSettings fileTransferSettings;
|
||||
|
||||
struct DownloadRequest
|
||||
struct FileTransferRequest
|
||||
{
|
||||
std::string uri;
|
||||
std::string expectedETag;
|
||||
bool verifyTLS = true;
|
||||
bool head = false;
|
||||
size_t tries = downloadSettings.tries;
|
||||
size_t tries = fileTransferSettings.tries;
|
||||
unsigned int baseRetryTimeMs = 250;
|
||||
ActivityId parentAct;
|
||||
bool decompress = true;
|
||||
|
|
@ -47,7 +47,7 @@ struct DownloadRequest
|
|||
std::string mimeType;
|
||||
std::function<void(char *, size_t)> dataCallback;
|
||||
|
||||
DownloadRequest(const std::string & uri)
|
||||
FileTransferRequest(const std::string & uri)
|
||||
: uri(uri), parentAct(getCurActivity()) { }
|
||||
|
||||
std::string verb()
|
||||
|
|
@ -56,7 +56,7 @@ struct DownloadRequest
|
|||
}
|
||||
};
|
||||
|
||||
struct DownloadResult
|
||||
struct FileTransferResult
|
||||
{
|
||||
bool cached = false;
|
||||
std::string etag;
|
||||
|
|
@ -65,74 +65,52 @@ struct DownloadResult
|
|||
uint64_t bodySize = 0;
|
||||
};
|
||||
|
||||
struct CachedDownloadRequest
|
||||
{
|
||||
std::string uri;
|
||||
bool unpack = false;
|
||||
std::string name;
|
||||
Hash expectedHash;
|
||||
unsigned int ttl;
|
||||
|
||||
CachedDownloadRequest(const std::string & uri);
|
||||
CachedDownloadRequest() = delete;
|
||||
};
|
||||
|
||||
struct CachedDownloadResult
|
||||
{
|
||||
// Note: 'storePath' may be different from 'path' when using a
|
||||
// chroot store.
|
||||
Path storePath;
|
||||
Path path;
|
||||
std::optional<std::string> etag;
|
||||
std::string effectiveUri;
|
||||
};
|
||||
|
||||
class Store;
|
||||
|
||||
struct Downloader
|
||||
struct FileTransfer
|
||||
{
|
||||
virtual ~Downloader() { }
|
||||
virtual ~FileTransfer() { }
|
||||
|
||||
/* Enqueue a download request, returning a future to the result of
|
||||
the download. The future may throw a DownloadError
|
||||
/* Enqueue a data transfer request, returning a future to the result of
|
||||
the download. The future may throw a FileTransferError
|
||||
exception. */
|
||||
virtual void enqueueDownload(const DownloadRequest & request,
|
||||
Callback<DownloadResult> callback) = 0;
|
||||
virtual void enqueueFileTransfer(const FileTransferRequest & request,
|
||||
Callback<FileTransferResult> callback) = 0;
|
||||
|
||||
std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
|
||||
std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
|
||||
|
||||
/* Synchronously download a file. */
|
||||
DownloadResult download(const DownloadRequest & request);
|
||||
FileTransferResult download(const FileTransferRequest & request);
|
||||
|
||||
/* Synchronously upload a file. */
|
||||
FileTransferResult upload(const FileTransferRequest & request);
|
||||
|
||||
/* Download a file, writing its data to a sink. The sink will be
|
||||
invoked on the thread of the caller. */
|
||||
void download(DownloadRequest && request, Sink & sink);
|
||||
|
||||
/* Check if the specified file is already in ~/.cache/nix/tarballs
|
||||
and is more recent than ‘tarball-ttl’ seconds. Otherwise,
|
||||
use the recorded ETag to verify if the server has a more
|
||||
recent version, and if so, download it to the Nix store. */
|
||||
CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
|
||||
void download(FileTransferRequest && request, Sink & sink);
|
||||
|
||||
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
|
||||
};
|
||||
|
||||
/* Return a shared Downloader object. Using this object is preferred
|
||||
/* Return a shared FileTransfer object. Using this object is preferred
|
||||
because it enables connection reuse and HTTP/2 multiplexing. */
|
||||
ref<Downloader> getDownloader();
|
||||
ref<FileTransfer> getFileTransfer();
|
||||
|
||||
/* Return a new Downloader object. */
|
||||
ref<Downloader> makeDownloader();
|
||||
/* Return a new FileTransfer object. */
|
||||
ref<FileTransfer> makeFileTransfer();
|
||||
|
||||
class DownloadError : public Error
|
||||
class FileTransferError : public Error
|
||||
{
|
||||
public:
|
||||
Downloader::Error error;
|
||||
DownloadError(Downloader::Error error, const FormatOrString & fs)
|
||||
FileTransfer::Error error;
|
||||
FileTransferError(FileTransfer::Error error, const FormatOrString & fs)
|
||||
: Error(fs), error(error)
|
||||
{ }
|
||||
};
|
||||
|
||||
bool isUri(const string & s);
|
||||
|
||||
/* Resolve deprecated 'channel:<foo>' URLs. */
|
||||
std::string resolveUri(const std::string & uri);
|
||||
|
||||
}
|
||||
|
|
@ -202,6 +202,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
|
|||
/* Read the `temproots' directory for per-process temporary root
|
||||
files. */
|
||||
for (auto & i : readDirectory(tempRootsDir)) {
|
||||
if (i.name[0] == '.') {
|
||||
// Ignore hidden files. Some package managers (notably portage) create
|
||||
// those to keep the directory alive.
|
||||
continue;
|
||||
}
|
||||
Path path = tempRootsDir + "/" + i.name;
|
||||
|
||||
pid_t pid = std::stoi(i.name);
|
||||
|
|
@ -414,7 +419,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
|||
|
||||
try {
|
||||
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
|
||||
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
|
||||
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n");
|
||||
for (const auto & line : mapLines) {
|
||||
auto match = std::smatch{};
|
||||
if (std::regex_match(line, match, mapRegex))
|
||||
|
|
@ -422,7 +427,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
|||
}
|
||||
|
||||
auto envFile = fmt("/proc/%s/environ", ent->d_name);
|
||||
auto envString = readFile(envFile, true);
|
||||
auto envString = readFile(envFile);
|
||||
auto env_end = std::sregex_iterator{};
|
||||
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
|
||||
unchecked[i->str()].emplace(envFile);
|
||||
|
|
@ -884,7 +889,7 @@ void LocalStore::autoGC(bool sync)
|
|||
if (statvfs(realStoreDir.c_str(), &st))
|
||||
throw SysError("getting filesystem info about '%s'", realStoreDir);
|
||||
|
||||
return (uint64_t) st.f_bavail * st.f_bsize;
|
||||
return (uint64_t) st.f_bavail * st.f_frsize;
|
||||
};
|
||||
|
||||
std::shared_future<void> future;
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ Settings::Settings()
|
|||
, nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
|
||||
, nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
|
||||
, nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
|
||||
, nixUserConfFiles(getUserConfigFiles())
|
||||
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
|
||||
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
|
||||
, nixManDir(canonPath(NIX_MAN_DIR))
|
||||
|
|
@ -77,13 +78,29 @@ void loadConfFile()
|
|||
~/.nix/nix.conf or the command line. */
|
||||
globalConfig.resetOverriden();
|
||||
|
||||
auto dirs = getConfigDirs();
|
||||
// Iterate over them in reverse so that the ones appearing first in the path take priority
|
||||
for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
|
||||
globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
|
||||
auto files = settings.nixUserConfFiles;
|
||||
for (auto file = files.rbegin(); file != files.rend(); file++) {
|
||||
globalConfig.applyConfigFile(*file);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Path> getUserConfigFiles()
|
||||
{
|
||||
// Use the paths specified in NIX_USER_CONF_FILES if it has been defined
|
||||
auto nixConfFiles = getEnv("NIX_USER_CONF_FILES");
|
||||
if (nixConfFiles.has_value()) {
|
||||
return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":");
|
||||
}
|
||||
|
||||
// Use the paths specified by the XDG spec
|
||||
std::vector<Path> files;
|
||||
auto dirs = getConfigDirs();
|
||||
for (auto & dir : dirs) {
|
||||
files.insert(files.end(), dir + "/nix/nix.conf");
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
unsigned int Settings::getDefaultCores()
|
||||
{
|
||||
return std::max(1U, std::thread::hardware_concurrency());
|
||||
|
|
@ -113,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
|
|||
void Settings::requireExperimentalFeature(const std::string & name)
|
||||
{
|
||||
if (!isExperimentalFeatureEnabled(name))
|
||||
throw Error("experimental Nix feature '%s' is disabled", name);
|
||||
throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
|
||||
}
|
||||
|
||||
bool Settings::isWSL1()
|
||||
|
|
@ -150,21 +167,24 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
|
|||
|
||||
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
|
||||
{
|
||||
args.mkFlag()
|
||||
.longName(name)
|
||||
.description("Enable sandboxing.")
|
||||
.handler([=](std::vector<std::string> ss) { override(smEnabled); })
|
||||
.category(category);
|
||||
args.mkFlag()
|
||||
.longName("no-" + name)
|
||||
.description("Disable sandboxing.")
|
||||
.handler([=](std::vector<std::string> ss) { override(smDisabled); })
|
||||
.category(category);
|
||||
args.mkFlag()
|
||||
.longName("relaxed-" + name)
|
||||
.description("Enable sandboxing, but allow builds to disable it.")
|
||||
.handler([=](std::vector<std::string> ss) { override(smRelaxed); })
|
||||
.category(category);
|
||||
args.addFlag({
|
||||
.longName = name,
|
||||
.description = "Enable sandboxing.",
|
||||
.category = category,
|
||||
.handler = {[=]() { override(smEnabled); }}
|
||||
});
|
||||
args.addFlag({
|
||||
.longName = "no-" + name,
|
||||
.description = "Disable sandboxing.",
|
||||
.category = category,
|
||||
.handler = {[=]() { override(smDisabled); }}
|
||||
});
|
||||
args.addFlag({
|
||||
.longName = "relaxed-" + name,
|
||||
.description = "Enable sandboxing, but allow builds to disable it.",
|
||||
.category = category,
|
||||
.handler = {[=]() { override(smRelaxed); }}
|
||||
});
|
||||
}
|
||||
|
||||
void MaxBuildJobsSetting::set(const std::string & str)
|
||||
|
|
|
|||
|
|
@ -53,9 +53,12 @@ public:
|
|||
/* The directory where state is stored. */
|
||||
Path nixStateDir;
|
||||
|
||||
/* The directory where configuration files are stored. */
|
||||
/* The directory where system configuration files are stored. */
|
||||
Path nixConfDir;
|
||||
|
||||
/* A list of user configuration files to load. */
|
||||
std::vector<Path> nixUserConfFiles;
|
||||
|
||||
/* The directory where internal helper programs are stored. */
|
||||
Path nixLibexecDir;
|
||||
|
||||
|
|
@ -351,12 +354,21 @@ public:
|
|||
Setting<Paths> pluginFiles{this, {}, "plugin-files",
|
||||
"Plugins to dynamically load at nix initialization time."};
|
||||
|
||||
Setting<std::string> githubAccessToken{this, "", "github-access-token",
|
||||
"GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
|
||||
|
||||
Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
|
||||
"Experimental Nix features to enable."};
|
||||
|
||||
bool isExperimentalFeatureEnabled(const std::string & name);
|
||||
|
||||
void requireExperimentalFeature(const std::string & name);
|
||||
|
||||
Setting<bool> allowDirty{this, true, "allow-dirty",
|
||||
"Whether to allow dirty Git/Mercurial trees."};
|
||||
|
||||
Setting<bool> warnDirty{this, true, "warn-dirty",
|
||||
"Whether to warn about dirty Git/Mercurial trees."};
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -369,6 +381,9 @@ void initPlugins();
|
|||
|
||||
void loadConfFile();
|
||||
|
||||
// Used by the Settings constructor
|
||||
std::vector<Path> getUserConfigFiles();
|
||||
|
||||
extern const string nixVersion;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#include "binary-cache-store.hh"
|
||||
#include "download.hh"
|
||||
#include "filetransfer.hh"
|
||||
#include "globals.hh"
|
||||
#include "nar-info-disk-cache.hh"
|
||||
|
||||
|
|
@ -85,14 +85,14 @@ protected:
|
|||
checkEnabled();
|
||||
|
||||
try {
|
||||
DownloadRequest request(cacheUri + "/" + path);
|
||||
FileTransferRequest request(cacheUri + "/" + path);
|
||||
request.head = true;
|
||||
getDownloader()->download(request);
|
||||
getFileTransfer()->download(request);
|
||||
return true;
|
||||
} catch (DownloadError & e) {
|
||||
} catch (FileTransferError & e) {
|
||||
/* S3 buckets return 403 if a file doesn't exist and the
|
||||
bucket is unlistable, so treat 403 as 404. */
|
||||
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
|
||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||
return false;
|
||||
maybeDisable();
|
||||
throw;
|
||||
|
|
@ -103,19 +103,19 @@ protected:
|
|||
const std::string & data,
|
||||
const std::string & mimeType) override
|
||||
{
|
||||
auto req = DownloadRequest(cacheUri + "/" + path);
|
||||
auto req = FileTransferRequest(cacheUri + "/" + path);
|
||||
req.data = std::make_shared<string>(data); // FIXME: inefficient
|
||||
req.mimeType = mimeType;
|
||||
try {
|
||||
getDownloader()->download(req);
|
||||
} catch (DownloadError & e) {
|
||||
getFileTransfer()->upload(req);
|
||||
} catch (FileTransferError & e) {
|
||||
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
|
||||
}
|
||||
}
|
||||
|
||||
DownloadRequest makeRequest(const std::string & path)
|
||||
FileTransferRequest makeRequest(const std::string & path)
|
||||
{
|
||||
DownloadRequest request(cacheUri + "/" + path);
|
||||
FileTransferRequest request(cacheUri + "/" + path);
|
||||
return request;
|
||||
}
|
||||
|
||||
|
|
@ -124,9 +124,9 @@ protected:
|
|||
checkEnabled();
|
||||
auto request(makeRequest(path));
|
||||
try {
|
||||
getDownloader()->download(std::move(request), sink);
|
||||
} catch (DownloadError & e) {
|
||||
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
|
||||
getFileTransfer()->download(std::move(request), sink);
|
||||
} catch (FileTransferError & e) {
|
||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
|
||||
maybeDisable();
|
||||
throw;
|
||||
|
|
@ -142,12 +142,12 @@ protected:
|
|||
|
||||
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
||||
|
||||
getDownloader()->enqueueDownload(request,
|
||||
{[callbackPtr, this](std::future<DownloadResult> result) {
|
||||
getFileTransfer()->enqueueFileTransfer(request,
|
||||
{[callbackPtr, this](std::future<FileTransferResult> result) {
|
||||
try {
|
||||
(*callbackPtr)(result.get().data);
|
||||
} catch (DownloadError & e) {
|
||||
if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
|
||||
} catch (FileTransferError & e) {
|
||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||
return (*callbackPtr)(std::shared_ptr<std::string>());
|
||||
maybeDisable();
|
||||
callbackPtr->rethrow();
|
||||
|
|
@ -174,4 +174,3 @@ static RegisterStoreImplementation regStore([](
|
|||
});
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ struct LegacySSHStore : public Store
|
|||
{ unsupported("queryPathFromHashPart"); }
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
bool recursive, HashType hashAlgo,
|
||||
FileIngestionMethod method, HashType hashAlgo,
|
||||
PathFilter & filter, RepairFlag repair) override
|
||||
{ unsupported("addToStore"); }
|
||||
|
||||
|
|
|
|||
|
|
@ -557,10 +557,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
|||
if (out == drv.outputs.end())
|
||||
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
|
||||
|
||||
bool recursive; Hash h;
|
||||
out->second.parseHashInfo(recursive, h);
|
||||
FileIngestionMethod method; Hash h;
|
||||
out->second.parseHashInfo(method, h);
|
||||
|
||||
check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
|
||||
check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
|
||||
}
|
||||
|
||||
else {
|
||||
|
|
@ -1043,11 +1043,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
|
||||
|
||||
StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
|
||||
bool recursive, HashType hashAlgo, RepairFlag repair)
|
||||
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
|
||||
{
|
||||
Hash h = hashString(hashAlgo, dump);
|
||||
|
||||
auto dstPath = makeFixedOutputPath(recursive, h, name);
|
||||
auto dstPath = makeFixedOutputPath(method, h, name);
|
||||
|
||||
addTempRoot(dstPath);
|
||||
|
||||
|
|
@ -1067,7 +1067,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
|||
|
||||
autoGC();
|
||||
|
||||
if (recursive) {
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
StringSource source(dump);
|
||||
restorePath(realPath, source);
|
||||
} else
|
||||
|
|
@ -1080,7 +1080,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
|||
above (if called with recursive == true and hashAlgo ==
|
||||
sha256); otherwise, compute it here. */
|
||||
HashResult hash;
|
||||
if (recursive) {
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
hash.first = hashAlgo == HashType::SHA256 ? h : hashString(HashType::SHA256, dump);
|
||||
hash.second = dump.size();
|
||||
} else
|
||||
|
|
@ -1091,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
|||
ValidPathInfo info(dstPath.clone());
|
||||
info.narHash = hash.first;
|
||||
info.narSize = hash.second;
|
||||
info.ca = makeFixedOutputCA(recursive, h);
|
||||
info.ca = makeFixedOutputCA(method, h);
|
||||
registerValidPath(info);
|
||||
}
|
||||
|
||||
|
|
@ -1103,7 +1103,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
|||
|
||||
|
||||
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
||||
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
Path srcPath(absPath(_srcPath));
|
||||
|
||||
|
|
@ -1111,12 +1111,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
|||
method for very large paths, but `copyPath' is mainly used for
|
||||
small files. */
|
||||
StringSink sink;
|
||||
if (recursive)
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
dumpPath(srcPath, sink, filter);
|
||||
else
|
||||
sink.s = make_ref<std::string>(readFile(srcPath));
|
||||
|
||||
return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair);
|
||||
return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ public:
|
|||
std::shared_ptr<FSAccessor> accessor) override;
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
bool recursive, HashType hashAlgo,
|
||||
FileIngestionMethod method, HashType hashAlgo,
|
||||
PathFilter & filter, RepairFlag repair) override;
|
||||
|
||||
/* Like addToStore(), but the contents of the path are contained
|
||||
|
|
@ -157,7 +157,7 @@ public:
|
|||
true) or simply the contents of a regular file (if recursive ==
|
||||
false). */
|
||||
StorePath addToStoreFromDump(const string & dump, const string & name,
|
||||
bool recursive = true, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) override;
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) override;
|
||||
|
||||
StorePath addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair) override;
|
||||
|
|
|
|||
|
|
@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1)
|
|||
libstore_LDFLAGS += -lseccomp
|
||||
endif
|
||||
|
||||
libstore_CXXFLAGS = \
|
||||
libstore_CXXFLAGS += \
|
||||
-I src/libutil -I src/libstore \
|
||||
-DNIX_PREFIX=\"$(prefix)\" \
|
||||
-DNIX_STORE_DIR=\"$(storedir)\" \
|
||||
-DNIX_DATA_DIR=\"$(datadir)\" \
|
||||
|
|
|
|||
|
|
@ -73,6 +73,11 @@ const size_t storePathHashLen = 32; // i.e. 160 bits
|
|||
/* Extension of derivations in the Nix store. */
|
||||
const std::string drvExtension = ".drv";
|
||||
|
||||
enum struct FileIngestionMethod : uint8_t {
|
||||
Flat = false,
|
||||
Recursive = true
|
||||
};
|
||||
|
||||
struct StorePathWithOutputs
|
||||
{
|
||||
StorePath path;
|
||||
|
|
|
|||
|
|
@ -484,7 +484,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
|
||||
|
||||
StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
|
||||
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
|
||||
|
||||
|
|
@ -492,10 +492,12 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
|
|||
|
||||
Path srcPath(absPath(_srcPath));
|
||||
|
||||
conn->to << wopAddToStore << name
|
||||
<< ((hashAlgo == HashType::SHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */
|
||||
<< (recursive ? 1 : 0)
|
||||
<< printHashType(hashAlgo);
|
||||
conn->to
|
||||
<< wopAddToStore
|
||||
<< name
|
||||
<< ((hashAlgo == HashType::SHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
|
||||
<< (method == FileIngestionMethod::Recursive ? 1 : 0)
|
||||
<< printHashType(hashAlgo);
|
||||
|
||||
try {
|
||||
conn->to.written = 0;
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ public:
|
|||
std::shared_ptr<FSAccessor> accessor) override;
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
bool recursive = true, HashType hashAlgo = HashType::SHA256,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256,
|
||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
|
||||
|
||||
StorePath addTextToStore(const string & name, const string & s,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
#include "nar-info-disk-cache.hh"
|
||||
#include "globals.hh"
|
||||
#include "compression.hh"
|
||||
#include "download.hh"
|
||||
#include "filetransfer.hh"
|
||||
#include "istringstream_nocopy.hh"
|
||||
|
||||
#include <aws/core/Aws.h>
|
||||
|
|
@ -132,7 +132,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
|
|||
return res;
|
||||
}
|
||||
|
||||
S3Helper::DownloadResult S3Helper::getObject(
|
||||
S3Helper::FileTransferResult S3Helper::getObject(
|
||||
const std::string & bucketName, const std::string & key)
|
||||
{
|
||||
debug("fetching 's3://%s/%s'...", bucketName, key);
|
||||
|
|
@ -146,7 +146,7 @@ S3Helper::DownloadResult S3Helper::getObject(
|
|||
return Aws::New<std::stringstream>("STRINGSTREAM");
|
||||
});
|
||||
|
||||
DownloadResult res;
|
||||
FileTransferResult res;
|
||||
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ struct S3Helper
|
|||
|
||||
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
|
||||
|
||||
struct DownloadResult
|
||||
struct FileTransferResult
|
||||
{
|
||||
std::shared_ptr<std::string> data;
|
||||
unsigned int durationMs;
|
||||
};
|
||||
|
||||
DownloadResult getObject(
|
||||
FileTransferResult getObject(
|
||||
const std::string & bucketName, const std::string & key);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include "thread-pool.hh"
|
||||
#include "json.hh"
|
||||
#include "derivations.hh"
|
||||
#include "url.hh"
|
||||
|
||||
#include <future>
|
||||
|
||||
|
|
@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const
|
|||
path = absPath(target, dirOf(path));
|
||||
}
|
||||
if (!isInStore(path))
|
||||
throw Error(format("path '%1%' is not in the Nix store") % path);
|
||||
throw NotInStore("path '%1%' is not in the Nix store", path);
|
||||
return path;
|
||||
}
|
||||
|
||||
|
|
@ -171,19 +172,22 @@ static std::string makeType(
|
|||
|
||||
|
||||
StorePath Store::makeFixedOutputPath(
|
||||
bool recursive,
|
||||
FileIngestionMethod recursive,
|
||||
const Hash & hash,
|
||||
std::string_view name,
|
||||
const StorePathSet & references,
|
||||
bool hasSelfReference) const
|
||||
{
|
||||
if (hash.type == HashType::SHA256 && recursive) {
|
||||
if (hash.type == HashType::SHA256 && recursive == FileIngestionMethod::Recursive) {
|
||||
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
|
||||
} else {
|
||||
assert(references.empty());
|
||||
return makeStorePath("output:out", hashString(HashType::SHA256,
|
||||
"fixed:out:" + (recursive ? (string) "r:" : "") +
|
||||
hash.to_string(Base::Base16) + ":"), name);
|
||||
return makeStorePath("output:out",
|
||||
hashString(HashType::SHA256,
|
||||
"fixed:out:"
|
||||
+ (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
|
||||
+ hash.to_string(Base::Base16) + ":"),
|
||||
name);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -200,10 +204,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
|
|||
|
||||
|
||||
std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
|
||||
const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
|
||||
const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const
|
||||
{
|
||||
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
|
||||
return std::make_pair(makeFixedOutputPath(recursive, h, name), h);
|
||||
Hash h = method == FileIngestionMethod::Recursive
|
||||
? hashPath(hashAlgo, srcPath, filter).first
|
||||
: hashFile(hashAlgo, srcPath);
|
||||
return std::make_pair(makeFixedOutputPath(method, h, name), h);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -781,8 +787,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
|
|||
}
|
||||
|
||||
else if (hasPrefix(ca, "fixed:")) {
|
||||
bool recursive = ca.compare(6, 2, "r:") == 0;
|
||||
Hash hash(std::string(ca, recursive ? 8 : 6));
|
||||
FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
|
||||
Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6));
|
||||
auto refs = cloneStorePathSet(references);
|
||||
bool hasSelfReference = false;
|
||||
if (refs.count(path)) {
|
||||
|
|
@ -826,9 +832,11 @@ Strings ValidPathInfo::shortRefs() const
|
|||
}
|
||||
|
||||
|
||||
std::string makeFixedOutputCA(bool recursive, const Hash & hash)
|
||||
std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
|
||||
{
|
||||
return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
|
||||
return "fixed:"
|
||||
+ (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
|
||||
+ hash.to_string();
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -866,27 +874,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
|
|||
Store::Params params;
|
||||
auto q = uri.find('?');
|
||||
if (q != std::string::npos) {
|
||||
for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
|
||||
auto e = s.find('=');
|
||||
if (e != std::string::npos) {
|
||||
auto value = s.substr(e + 1);
|
||||
std::string decoded;
|
||||
for (size_t i = 0; i < value.size(); ) {
|
||||
if (value[i] == '%') {
|
||||
if (i + 2 >= value.size())
|
||||
throw Error("invalid URI parameter '%s'", value);
|
||||
try {
|
||||
decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
|
||||
i += 3;
|
||||
} catch (...) {
|
||||
throw Error("invalid URI parameter '%s'", value);
|
||||
}
|
||||
} else
|
||||
decoded += value[i++];
|
||||
}
|
||||
params[s.substr(0, e)] = decoded;
|
||||
}
|
||||
}
|
||||
params = decodeQuery(uri.substr(q + 1));
|
||||
uri = uri_.substr(0, q);
|
||||
}
|
||||
return {uri, params};
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ MakeError(InvalidPath, Error);
|
|||
MakeError(Unsupported, Error);
|
||||
MakeError(SubstituteGone, Error);
|
||||
MakeError(SubstituterDisabled, Error);
|
||||
MakeError(NotInStore, Error);
|
||||
|
||||
|
||||
struct BasicDerivation;
|
||||
|
|
@ -43,7 +44,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
|
|||
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
|
||||
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
|
||||
|
||||
|
||||
/* Magic header of exportPath() output (obsolete). */
|
||||
const uint32_t exportMagic = 0x4558494e;
|
||||
|
||||
|
|
@ -346,7 +346,7 @@ public:
|
|||
StorePath makeOutputPath(const string & id,
|
||||
const Hash & hash, std::string_view name) const;
|
||||
|
||||
StorePath makeFixedOutputPath(bool recursive,
|
||||
StorePath makeFixedOutputPath(FileIngestionMethod method,
|
||||
const Hash & hash, std::string_view name,
|
||||
const StorePathSet & references = {},
|
||||
bool hasSelfReference = false) const;
|
||||
|
|
@ -358,7 +358,7 @@ public:
|
|||
store path to which srcPath is to be copied. Returns the store
|
||||
path and the cryptographic hash of the contents of srcPath. */
|
||||
std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
|
||||
const Path & srcPath, bool recursive = true,
|
||||
const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive,
|
||||
HashType hashAlgo = HashType::SHA256, PathFilter & filter = defaultPathFilter) const;
|
||||
|
||||
/* Preparatory part of addTextToStore().
|
||||
|
|
@ -462,12 +462,12 @@ public:
|
|||
The function object `filter' can be used to exclude files (see
|
||||
libutil/archive.hh). */
|
||||
virtual StorePath addToStore(const string & name, const Path & srcPath,
|
||||
bool recursive = true, HashType hashAlgo = HashType::SHA256,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256,
|
||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
|
||||
|
||||
// FIXME: remove?
|
||||
virtual StorePath addToStoreFromDump(const string & dump, const string & name,
|
||||
bool recursive = true, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair)
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair)
|
||||
{
|
||||
throw Error("addToStoreFromDump() is not supported by this store");
|
||||
}
|
||||
|
|
@ -850,7 +850,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
|
|||
|
||||
/* Compute the content-addressability assertion (ValidPathInfo::ca)
|
||||
for paths created by makeFixedOutputPath() / addToStore(). */
|
||||
std::string makeFixedOutputCA(bool recursive, const Hash & hash);
|
||||
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
|
||||
|
||||
|
||||
/* Split URI into protocol+hierarchy part and its parameter set. */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue