nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-24 03:09:35 +01:00
Code Activity

Merge commit 'df552ff53e' into progress-bar

Browse source
This commit is contained in:
John Ericson 2023-03-11 17:03:54 -05:00
commit 38949e6be4
259 changed files with 3960 additions and 29251 deletions
Download patch file Download diff file Expand all files Collapse all files

61
src/libstore/binary-cache-store.cc
View file

@ -31,7 +31,7 @@ BinaryCacheStore::BinaryCacheStore(const Params & params)
StringSink sink;
sink << narVersionMagic1;
narMagic = *sink.s;
narMagic = sink.s;
}
void BinaryCacheStore::init()
@ -68,7 +68,7 @@ void BinaryCacheStore::upsertFile(const std::string & path,
}
void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept
Callback<std::optional<std::string>> callback) noexcept
{
try {
callback(getFile(path));
@ -77,9 +77,9 @@ void BinaryCacheStore::getFile(const std::string & path,
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
{
std::promise<std::shared_ptr<std::string>> promise;
std::promise<std::optional<std::string>> promise;
getFile(path,
{[&](std::future<std::shared_ptr<std::string>> result) {
{[&](std::future<std::optional<std::string>> result) {
try {
promise.set_value(result.get());
} catch (...) {
@ -89,15 +89,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
sink(*promise.get_future().get());
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
std::optional<std::string> BinaryCacheStore::getFile(const std::string & path)
{
StringSink sink;
try {
getFile(path, sink);
} catch (NoSuchBinaryCacheFile &) {
return nullptr;
return std::nullopt;
}
return sink.s;
return std::move(sink.s);
}
std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
@ -307,7 +307,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
}});
}
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
{
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
@ -367,11 +367,11 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getFile(narInfoFile,
{[=](std::future<std::shared_ptr<std::string>> fut) {
{[=](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
if (!data) return (*callbackPtr)(nullptr);
if (!data) return (*callbackPtr)({});
stats.narInfoRead++;
@ -385,8 +385,14 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
}});
}
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references)
StorePath BinaryCacheStore::addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references)
{
/* FIXME: Make BinaryCacheStore::addToStoreCommon support
non-recursive+sha256 so we can just use the default
@ -418,8 +424,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
})->path;
}
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
StorePath BinaryCacheStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair)
{
auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, textHash, references);
@ -429,7 +438,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
StringSource source(sink.s);
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first };
info.narSize = nar.second;
@ -446,11 +455,11 @@ void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
Callback<std::shared_ptr<std::string>> newCallback = {
[=](std::future<std::shared_ptr<std::string>> fut) {
Callback<std::optional<std::string>> newCallback = {
[=](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
if (!data) return (*callbackPtr)(nullptr);
if (!data) return (*callbackPtr)({});
auto realisation = Realisation::fromJSON(
nlohmann::json::parse(*data), outputInfoFilePath);
@ -490,7 +499,7 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
writeNarInfo(narInfo);
}
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
{
auto drvPath = path;
@ -498,10 +507,10 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (!info->deriver) return nullptr;
if (!info->deriver) return std::nullopt;
drvPath = *info->deriver;
} catch (InvalidPath &) {
return nullptr;
return std::nullopt;
}
}
@ -512,4 +521,14 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
return getFile(logPath);
}
void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
assert(drvPath.isDerivation());
upsertFile(
"log/" + std::string(drvPath.to_string()),
(std::string) log, // FIXME: don't copy
"text/plain; charset=utf-8");
}
}

34
src/libstore/binary-cache-store.hh
View file

@ -51,6 +51,7 @@ public:
const std::string & mimeType) = 0;
void upsertFile(const std::string & path,
// FIXME: use std::string_view
std::string && data,
const std::string & mimeType);
@ -62,10 +63,11 @@ public:
/* Fetch the specified file and call the specified callback with
the result. A subclass may implement this asynchronously. */
virtual void getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept;
virtual void getFile(
const std::string & path,
Callback<std::optional<std::string>> callback) noexcept;
std::shared_ptr<std::string> getFile(const std::string & path);
std::optional<std::string> getFile(const std::string & path);
public:
@ -96,15 +98,23 @@ public:
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references ) override;
StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair, const StorePathSet & references) override;
StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
@ -117,7 +127,9 @@ public:
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
std::optional<std::string> getBuildLog(const StorePath & path) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
};

18
src/libstore/build/derivation-goal.cc
View file

@ -116,7 +116,7 @@ DerivationGoal::~DerivationGoal()
}
string DerivationGoal::key()
std::string DerivationGoal::key()
{
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before
@ -278,7 +278,7 @@ void DerivationGoal::outputsSubstitutionTried()
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
done(BuildResult::TransientFailure,
fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
worker.store.printStorePath(drvPath)));
return;
}
@ -1024,7 +1024,7 @@ HookReply DerivationGoal::tryBuildHook()
/* Read the first line of input, which should be a word indicating
whether the hook wishes to perform the build. */
string reply;
std::string reply;
while (true) {
auto s = [&]() {
try {
@ -1036,8 +1036,8 @@ HookReply DerivationGoal::tryBuildHook()
}();
if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
;
else if (string(s, 0, 2) == "# ") {
reply = string(s, 2);
else if (s.substr(0, 2) == "# ") {
reply = s.substr(2);
break;
}
else {
@ -1102,7 +1102,7 @@ HookReply DerivationGoal::tryBuildHook()
/* Create the log file and pipe. */
Path logFile = openLogFile();
set<int> fds;
std::set<int> fds;
fds.insert(hook->fromHook.readSide.get());
fds.insert(hook->builderOut.readSide.get());
worker.childStarted(shared_from_this(), fds, false, false);
@ -1151,10 +1151,10 @@ Path DerivationGoal::openLogFile()
logDir = localStore->logDir;
else
logDir = settings.nixLogDir;
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2));
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
createDirs(dir);
Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2),
settings.compressLog ? ".bz2" : "");
fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
@ -1187,7 +1187,7 @@ bool DerivationGoal::isReadDesc(int fd)
}
void DerivationGoal::handleChildOutput(int fd, const string & data)
void DerivationGoal::handleChildOutput(int fd, std::string_view data)
{
if (isReadDesc(fd))
{

4
src/libstore/build/derivation-goal.hh
View file

@ -150,7 +150,7 @@ struct DerivationGoal : public Goal
void timedOut(Error && ex) override;
string key() override;
std::string key() override;
void work() override;
@ -205,7 +205,7 @@ struct DerivationGoal : public Goal
virtual bool isReadDesc(int fd);
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;
void flushLine();

2
src/libstore/build/drv-output-substitution-goal.cc
View file

@ -137,7 +137,7 @@ void DrvOutputSubstitutionGoal::finished()
amDone(ecSuccess);
}
string DrvOutputSubstitutionGoal::key()
std::string DrvOutputSubstitutionGoal::key()
{
/* "a$" ensures substitution goals happen before derivation
goals. */

2
src/libstore/build/drv-output-substitution-goal.hh
View file

@ -51,7 +51,7 @@ public:
void timedOut(Error && ex) override { abort(); };
string key() override;
std::string key() override;
void work() override;
void handleEOF(int fd) override;

4
src/libstore/build/goal.cc
View file

@ -5,8 +5,8 @@ namespace nix {
bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
string s1 = a->key();
string s2 = b->key();
std::string s1 = a->key();
std::string s2 = b->key();
return s1 < s2;
}

12
src/libstore/build/goal.hh
View file

@ -18,8 +18,8 @@ struct CompareGoalPtrs {
};
/* Set of goals. */
typedef set<GoalPtr, CompareGoalPtrs> Goals;
typedef set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
typedef std::set<GoalPtr, CompareGoalPtrs> Goals;
typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
/* A map of paths to goals (and the other way around). */
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
@ -50,7 +50,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
unsigned int nrIncompleteClosure;
/* Name of this goal for debugging purposes. */
string name;
std::string name;
/* Whether the goal is finished. */
ExitCode exitCode;
@ -75,7 +75,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
virtual void handleChildOutput(int fd, const string & data)
virtual void handleChildOutput(int fd, std::string_view data)
{
abort();
}
@ -87,7 +87,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
void trace(const FormatOrString & fs);
string getName()
std::string getName()
{
return name;
}
@ -97,7 +97,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
by the worker (important!), etc. */
virtual void timedOut(Error && ex) = 0;
virtual string key() = 0;
virtual std::string key() = 0;
void amDone(ExitCode result, std::optional<Error> ex = {});

83
src/libstore/build/local-derivation-goal.cc
View file

@ -481,12 +481,12 @@ void LocalDerivationGoal::startBuilder()
temporary build directory. The text files have the format used
by `nix-store --register-validity'. However, the deriver
fields are left empty. */
string s = get(drv->env, "exportReferencesGraph").value_or("");
auto s = get(drv->env, "exportReferencesGraph").value_or("");
Strings ss = tokenizeString<Strings>(s);
if (ss.size() % 2 != 0)
throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
string fileName = *i++;
auto fileName = *i++;
static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
if (!std::regex_match(fileName, regex))
throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
@ -517,10 +517,10 @@ void LocalDerivationGoal::startBuilder()
i.pop_back();
}
size_t p = i.find('=');
if (p == string::npos)
if (p == std::string::npos)
dirsInChroot[i] = {i, optional};
else
dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional};
}
dirsInChroot[tmpDirInSandbox] = tmpDir;
@ -671,9 +671,10 @@ void LocalDerivationGoal::startBuilder()
auto state = stBegin;
auto lines = runProgram(settings.preBuildHook, false, args);
auto lastPos = std::string::size_type{0};
for (auto nlPos = lines.find('\n'); nlPos != string::npos;
nlPos = lines.find('\n', lastPos)) {
auto line = std::string{lines, lastPos, nlPos - lastPos};
for (auto nlPos = lines.find('\n'); nlPos != std::string::npos;
nlPos = lines.find('\n', lastPos))
{
auto line = lines.substr(lastPos, nlPos - lastPos);
lastPos = nlPos + 1;
if (state == stBegin) {
if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
@ -686,10 +687,10 @@ void LocalDerivationGoal::startBuilder()
state = stBegin;
} else {
auto p = line.find('=');
if (p == string::npos)
if (p == std::string::npos)
dirsInChroot[line] = line;
else
dirsInChroot[string(line, 0, p)] = string(line, p + 1);
dirsInChroot[line.substr(0, p)] = line.substr(p + 1);
}
}
}
@ -913,9 +914,12 @@ void LocalDerivationGoal::startBuilder()
sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
if (sandboxMountNamespace.get() == -1)
throw SysError("getting sandbox mount namespace");
sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY);
if (sandboxUserNamespace.get() == -1)
throw SysError("getting sandbox user namespace");
if (usingUserNamespace) {
sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY);
if (sandboxUserNamespace.get() == -1)
throw SysError("getting sandbox user namespace");
}
/* Signal the builder that we've updated its user namespace. */
writeFull(userNamespaceSync.writeSide.get(), "1");
@ -939,7 +943,7 @@ void LocalDerivationGoal::startBuilder()
/* Check if setting up the build environment failed. */
std::vector<std::string> msgs;
while (true) {
string msg = [&]() {
std::string msg = [&]() {
try {
return readLine(builderOut.readSide.get());
} catch (Error & e) {
@ -951,8 +955,8 @@ void LocalDerivationGoal::startBuilder()
throw;
}
}();
if (string(msg, 0, 1) == "\2") break;
if (string(msg, 0, 1) == "\1") {
if (msg.substr(0, 1) == "\2") break;
if (msg.substr(0, 1) == "\1") {
FdSource source(builderOut.readSide.get());
auto ex = readError(source);
ex.addTrace({}, "while setting up the build environment");
@ -988,7 +992,7 @@ void LocalDerivationGoal::initTmpDir() {
env[i.first] = i.second;
} else {
auto hash = hashString(htSHA256, i.first);
string fn = ".attr-" + hash.to_string(Base32, false);
std::string fn = ".attr-" + hash.to_string(Base32, false);
Path p = tmpDir + "/" + fn;
writeFile(p, rewriteStrings(i.second, inputRewrites));
chownToBuilder(p);
@ -1079,7 +1083,7 @@ void LocalDerivationGoal::writeStructuredAttrs()
for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
/* The placeholder must have a rewrite, so we use it to cover both the
cases where we know or don't know the output path ahead of time. */
rewritten[i] = rewriteStrings(v, inputRewrites);
rewritten[i] = rewriteStrings((std::string) v, inputRewrites);
}
json["outputs"] = rewritten;
@ -1185,10 +1189,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ throw Error("queryPathFromHashPart"); }
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair,
const StorePathSet & references = StorePathSet()) override
StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override
{ throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
@ -1198,17 +1206,24 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
goal.addDependency(info.path);
}
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair = NoRepair) override
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair = NoRepair) override
{
auto path = next->addTextToStore(name, s, references, repair);
goal.addDependency(path);
return path;
}
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair,
const StorePathSet & references = StorePathSet()) override
StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
FileIngestionMethod method,
HashType hashAlgo,
RepairFlag repair,
const StorePathSet & references) override
{
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
goal.addDependency(path);
@ -1428,7 +1443,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
Path source = worker.store.Store::toRealPath(path);
Path target = chrootRootDir + worker.store.printStorePath(path);
debug("bind-mounting %s -> %s", source, target);
debug("bind-mounting %s -> %s", target, source);
if (pathExists(target))
throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
@ -1990,7 +2005,7 @@ void LocalDerivationGoal::runChild()
args.push_back(rewriteStrings(i, inputRewrites));
/* Indicate that we managed to set up the build environment. */
writeFull(STDERR_FILENO, string("\2\n"));
writeFull(STDERR_FILENO, std::string("\2\n"));
/* Execute the program. This should not return. */
if (drv->isBuiltin()) {
@ -2008,7 +2023,7 @@ void LocalDerivationGoal::runChild()
else if (drv->builder == "builtin:unpack-channel")
builtinUnpackChannel(drv2);
else
throw Error("unsupported builtin builder '%1%'", string(drv->builder, 8));
throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8));
_exit(0);
} catch (std::exception & e) {
writeFull(STDERR_FILENO, e.what() + std::string("\n"));
@ -2227,8 +2242,8 @@ void LocalDerivationGoal::registerOutputs()
StringSink sink;
dumpPath(actualPath, sink);
deletePath(actualPath);
sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
StringSource source(*sink.s);
sink.s = rewriteStrings(sink.s, outputRewrites);
StringSource source(sink.s);
restorePath(actualPath, source);
}
};
@ -2296,7 +2311,7 @@ void LocalDerivationGoal::registerOutputs()
StringSink sink;
dumpPath(actualPath, sink);
RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink);
rsink2(*sink.s);
rsink2(sink.s);
rsink2.flush();
});
Path tmpPath = actualPath + ".tmp";
@ -2468,7 +2483,7 @@ void LocalDerivationGoal::registerOutputs()
}
if (curRound == nrRounds) {
localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences()
localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences()
worker.markContentsGood(newInfo.path);
}
@ -2692,7 +2707,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
}
if (!badPaths.empty()) {
string badPathsStr;
std::string badPathsStr;
for (auto & i : badPaths) {
badPathsStr += "\n ";
badPathsStr += worker.store.printStorePath(i);

2
src/libstore/build/local-derivation-goal.hh
View file

@ -58,7 +58,7 @@ struct LocalDerivationGoal : public DerivationGoal
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
DirsInChroot dirsInChroot;
typedef map<string, string> Environment;
typedef map<std::string, std::string> Environment;
Environment env;
#if __APPLE__

2
src/libstore/build/substitution-goal.cc
View file

@ -284,7 +284,7 @@ void PathSubstitutionGoal::finished()
}
void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data)
{
}

4
src/libstore/build/substitution-goal.hh
View file

@ -62,7 +62,7 @@ public:
void timedOut(Error && ex) override { abort(); };
string key() override
std::string key() override
{
/* "a$" ensures substitution goals happen before derivation
goals. */
@ -80,7 +80,7 @@ public:
void finished();
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;
void cleanup() override;

6
src/libstore/build/worker.cc
View file

@ -165,7 +165,7 @@ unsigned Worker::getNrLocalBuilds()
}
void Worker::childStarted(GoalPtr goal, const set<int> & fds,
void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
bool inBuildSlot, bool respectTimeouts)
{
Child child;
@ -387,7 +387,7 @@ void Worker::waitForInput()
GoalPtr goal = j->goal.lock();
assert(goal);
set<int> fds2(j->fds);
std::set<int> fds2(j->fds);
std::vector<unsigned char> buffer(4096);
for (auto & k : fds2) {
if (pollStatus.at(fdToPollStatus.at(k)).revents) {
@ -404,7 +404,7 @@ void Worker::waitForInput()
} else {
printMsg(lvlVomit, "%1%: read %2% bytes",
goal->getName(), rd);
string data((char *) buffer.data(), rd);
std::string data((char *) buffer.data(), rd);
j->lastOutput = after;
goal->handleChildOutput(k, data);
}

4
src/libstore/build/worker.hh
View file

@ -38,7 +38,7 @@ struct Child
{
WeakGoalPtr goal;
Goal * goal2; // ugly hackery
set<int> fds;
std::set<int> fds;
bool respectTimeouts;
bool inBuildSlot;
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
@ -167,7 +167,7 @@ public:
/* Registers a running child process. `inBuildSlot' means that
the process counts towards the jobs limit. */
void childStarted(GoalPtr goal, const set<int> & fds,
void childStarted(GoalPtr goal, const std::set<int> & fds,
bool inBuildSlot, bool respectTimeouts);
/* Unregisters a running child process. `wakeSleepers' should be

4
src/libstore/builtins/buildenv.cc
View file

@ -123,7 +123,7 @@ void buildProfile(const Path & out, Packages && pkgs)
createLinks(state, pkgDir, out, priority);
try {
for (const auto & p : tokenizeString<std::vector<string>>(
for (const auto & p : tokenizeString<std::vector<std::string>>(
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
if (!done.count(p))
postponed.insert(p);
@ -161,7 +161,7 @@ void buildProfile(const Path & out, Packages && pkgs)
void builtinBuildenv(const BasicDerivation & drv)
{
auto getAttr = [&](const string & name) {
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;

2
src/libstore/builtins/fetchurl.cc
View file

@ -16,7 +16,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
writeFile(settings.netrcFile, netrcData, 0600);
}
auto getAttr = [&](const string & name) {
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;

2
src/libstore/builtins/unpack-channel.cc
View file

@ -5,7 +5,7 @@ namespace nix {
void builtinUnpackChannel(const BasicDerivation & drv)
{
auto getAttr = [&](const string & name) {
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;

54
src/libstore/daemon.cc
View file

@ -69,7 +69,7 @@ struct TunnelLogger : public Logger
StringSink buf;
buf << STDERR_NEXT << (fs.s + "\n");
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
void logEI(const ErrorInfo & ei) override
@ -81,7 +81,7 @@ struct TunnelLogger : public Logger
StringSink buf;
buf << STDERR_NEXT << oss.str();
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
/* startWork() means that we're starting an operation for which we
@ -129,7 +129,7 @@ struct TunnelLogger : public Logger
StringSink buf;
buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent;
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
void stopActivity(ActivityId act) override
@ -137,7 +137,7 @@ struct TunnelLogger : public Logger
if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
StringSink buf;
buf << STDERR_STOP_ACTIVITY << act;
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
void result(ActivityId act, ResultType type, const Fields & fields) override
@ -145,7 +145,7 @@ struct TunnelLogger : public Logger
if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
StringSink buf;
buf << STDERR_RESULT << act << type << fields;
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
};
@ -468,17 +468,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
dontCheckSigs = false;
logger->startWork();
FramedSource source(from);
store->addMultipleToStore(source,
RepairFlag{repair},
dontCheckSigs ? NoCheckSigs : CheckSigs);
{
FramedSource source(from);
store->addMultipleToStore(source,
RepairFlag{repair},
dontCheckSigs ? NoCheckSigs : CheckSigs);
}
logger->stopWork();
break;
}
case wopAddTextToStore: {
string suffix = readString(from);
string s = readString(from);
std::string suffix = readString(from);
std::string s = readString(from);
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
logger->startWork();
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
@ -696,8 +698,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
unsigned int n = readInt(from);
for (unsigned int i = 0; i < n; i++) {
string name = readString(from);
string value = readString(from);
auto name = readString(from);
auto value = readString(from);
clientSettings.overrides.emplace(name, value);
}
}
@ -852,14 +854,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
else {
std::unique_ptr<Source> source;
StringSink saved;
if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to);
else {
StringSink saved;
TeeSource tee { from, saved };
ParseSink ether;
parseDump(ether, tee);
source = std::make_unique<StringSource>(std::move(*saved.s));
source = std::make_unique<StringSource>(saved.s);
}
logger->startWork();
@ -920,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
case wopAddBuildLog: {
StorePath path{readString(from)};
logger->startWork();
if (!trusted)
throw Error("you are not privileged to add logs");
{
FramedSource source(from);
StringSink sink;
source.drainInto(sink);
store->addBuildLog(path, sink.s);
}
logger->stopWork();
to << 1;
break;
}
default:
throw Error("invalid operation %1%", op);
}
@ -963,7 +981,11 @@ void processConnection(
readInt(from);
}
readInt(from); // obsolete reserveSpace
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
readInt(from); // obsolete reserveSpace
if (GET_PROTOCOL_MINOR(clientVersion) >= 33)
to << nixVersion;
/* Send startup error messages to the client. */
tunnelLogger->startWork();

41
src/libstore/derivations.cc
View file

@ -4,6 +4,7 @@
#include "util.hh"
#include "worker-protocol.hh"
#include "fs-accessor.hh"
#include <boost/container/small_vector.hpp>
namespace nix {
@ -81,7 +82,7 @@ bool derivationIsImpure(DerivationType dt) {
bool BasicDerivation::isBuiltin() const
{
return string(builder, 0, 8) == "builtin:";
return builder.substr(0, 8) == "builtin:";
}
@ -103,19 +104,19 @@ StorePath writeDerivation(Store & store,
/* Read string `s' from stream `str'. */
static void expect(std::istream & str, const string & s)
static void expect(std::istream & str, std::string_view s)
{
char s2[s.size()];
str.read(s2, s.size());
if (string(s2, s.size()) != s)
if (std::string(s2, s.size()) != s)
throw FormatError("expected string '%1%'", s);
}
/* Read a C-style string from stream `str'. */
static string parseString(std::istream & str)
static std::string parseString(std::istream & str)
{
string res;
std::string res;
expect(str, "\"");
int c;
while ((c = str.get()) != '"')
@ -171,7 +172,7 @@ static DerivationOutput parseDerivationOutput(const Store & store,
{
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
if (hashAlgo.substr(0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = hashAlgo.substr(2);
}
@ -259,8 +260,8 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
/* Parse the environment variables. */
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "("); string name = parseString(str);
expect(str, ","); string value = parseString(str);
expect(str, "("); auto name = parseString(str);
expect(str, ","); auto value = parseString(str);
expect(str, ")");
drv.env[name] = value;
}
@ -270,9 +271,11 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
}
static void printString(string & res, std::string_view s)
static void printString(std::string & res, std::string_view s)
{
char buf[s.size() * 2 + 2];
boost::container::small_vector<char, 64 * 1024> buffer;
buffer.reserve(s.size() * 2 + 2);
char * buf = buffer.data();
char * p = buf;
*p++ = '"';
for (auto c : s)
@ -286,7 +289,7 @@ static void printString(string & res, std::string_view s)
}
static void printUnquotedString(string & res, std::string_view s)
static void printUnquotedString(std::string & res, std::string_view s)
{
res += '"';
res.append(s);
@ -295,7 +298,7 @@ static void printUnquotedString(string & res, std::string_view s)
template<class ForwardIterator>
static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
static void printStrings(std::string & res, ForwardIterator i, ForwardIterator j)
{
res += '[';
bool first = true;
@ -308,7 +311,7 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
template<class ForwardIterator>
static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterator j)
static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIterator j)
{
res += '[';
bool first = true;
@ -320,10 +323,10 @@ static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterato
}
string Derivation::unparse(const Store & store, bool maskOutputs,
std::string Derivation::unparse(const Store & store, bool maskOutputs,
std::map<std::string, StringSet> * actualInputs) const
{
string s;
std::string s;
s.reserve(65536);
s += "Derive([";
@ -398,7 +401,7 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
// FIXME: remove
bool isDerivation(const string & fileName)
bool isDerivation(const std::string & fileName)
{
return hasSuffix(fileName, drvExtension);
}
@ -590,7 +593,7 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
}
bool wantOutput(const string & output, const std::set<string> & wanted)
bool wantOutput(const std::string & output, const std::set<std::string> & wanted)
{
return wanted.empty() || wanted.find(output) != wanted.end();
}
@ -696,10 +699,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
}
std::string hashPlaceholder(const std::string & outputName)
std::string hashPlaceholder(const std::string_view outputName)
{
// FIXME: memoize?
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
}
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)

14
src/libstore/derivations.hh
View file

@ -59,21 +59,19 @@ struct DerivationOutput
std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
typedef std::map<std::string, DerivationOutput> DerivationOutputs;
/* These are analogues to the previous DerivationOutputs data type, but they
also contains, for each output, the (optional) store path in which it would
be written. To calculate values of these types, see the corresponding
functions in BasicDerivation */
typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>>
typedef std::map<std::string, std::pair<DerivationOutput, std::optional<StorePath>>>
DerivationOutputsAndOptPaths;
/* For inputs that are sub-derivations, we specify exactly which
output IDs we are interested in. */
typedef std::map<StorePath, StringSet> DerivationInputs;
typedef std::map<string, string> StringPairs;
enum struct DerivationType : uint8_t {
InputAddressed,
DeferredInputAddressed,
@ -103,7 +101,7 @@ struct BasicDerivation
{
DerivationOutputs outputs; /* keyed on symbolic IDs */
StorePathSet inputSrcs; /* inputs that are sources */
string platform;
std::string platform;
Path builder;
Strings args;
StringPairs env;
@ -164,7 +162,7 @@ StorePath writeDerivation(Store & store,
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
// FIXME: remove
bool isDerivation(const string & fileName);
bool isDerivation(const std::string & fileName);
/* Calculate the name that will be used for the store path for this
output.
@ -222,7 +220,7 @@ typedef std::map<StorePath, DrvHashModulo> DrvHashes;
// FIXME: global, though at least thread-safe.
extern Sync<DrvHashes> drvHashes;
bool wantOutput(const string & output, const std::set<string> & wanted);
bool wantOutput(const std::string & output, const std::set<std::string> & wanted);
struct Source;
struct Sink;
@ -236,7 +234,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
It is used as a placeholder to allow derivations to refer to their
own outputs without needing to use the hash of a derivation in
itself, making the hash near-impossible to calculate. */
std::string hashPlaceholder(const std::string & outputName);
std::string hashPlaceholder(const std::string_view outputName);
/* This creates an opaque and almost certainly unique string
deterministically from a derivation path and output name.

4
src/libstore/derived-path.cc
View file

@ -75,9 +75,9 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi
assert(n != s.npos);
auto drvPath = store.parseStorePath(s.substr(0, n));
auto outputsS = s.substr(n + 1);
std::set<string> outputs;
std::set<std::string> outputs;
if (outputsS != "*")
outputs = tokenizeString<std::set<string>>(outputsS, ",");
outputs = tokenizeString<std::set<std::string>>(outputsS, ",");
return {drvPath, outputs};
}

9
src/libstore/dummy-store.cc
View file

@ -21,7 +21,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
, Store(params)
{ }
string getUri() override
std::string getUri() override
{
return *uriSchemes().begin();
}
@ -43,8 +43,11 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
RepairFlag repair, CheckSigsFlag checkSigs) override
{ unsupported("addToStore"); }
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override
{ unsupported("addTextToStore"); }
void narFromPath(const StorePath & path, Sink & sink) override

6
src/libstore/export-import.cc
View file

@ -75,20 +75,20 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
auto deriver = readString(source);
auto narHash = hashString(htSHA256, *saved.s);
auto narHash = hashString(htSHA256, saved.s);
ValidPathInfo info { path, narHash };
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.references = references;
info.narSize = saved.s->size();
info.narSize = saved.s.size();
// Ignore optional legacy signature.
if (readInt(source) == 1)
readString(source);
// Can't use underlying source, which would have been exhausted
auto source = StringSource { *saved.s };
auto source = StringSource(saved.s);
addToStore(info, source, NoRepair, checkSigs);
res.push_back(info.path);

50
src/libstore/filetransfer.cc
View file

@ -33,12 +33,12 @@ FileTransferSettings fileTransferSettings;
static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
std::string resolveUri(const std::string & uri)
std::string resolveUri(std::string_view uri)
{
if (uri.compare(0, 8, "channel:") == 0)
return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz";
return "https://nixos.org/channels/" + std::string(uri.substr(8)) + "/nixexprs.tar.xz";
else
return uri;
return std::string(uri);
}
struct curlFileTransfer : public FileTransfer
@ -106,7 +106,7 @@ struct curlFileTransfer : public FileTransfer
this->request.dataCallback(data);
}
} else
this->result.data->append(data);
this->result.data.append(data);
})
{
if (!request.expectedETag.empty())
@ -128,7 +128,7 @@ struct curlFileTransfer : public FileTransfer
if (requestHeaders) curl_slist_free_all(requestHeaders);
try {
if (!done)
fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri));
fail(FileTransferError(Interrupted, {}, "download of '%s' was interrupted", request.uri));
} catch (...) {
ignoreException();
}
@ -195,17 +195,17 @@ struct curlFileTransfer : public FileTransfer
std::smatch match;
if (std::regex_match(line, match, statusLine)) {
result.etag = "";
result.data = std::make_shared<std::string>();
result.data.clear();
result.bodySize = 0;
statusMsg = trim(match[1]);
statusMsg = trim((std::string &) match[1]);
acceptRanges = false;
encoding = "";
} else {
auto i = line.find(':');
if (i != string::npos) {
string name = toLower(trim(string(line, 0, i)));
if (i != std::string::npos) {
std::string name = toLower(trim(line.substr(0, i)));
if (name == "etag") {
result.etag = trim(string(line, i + 1));
result.etag = trim(line.substr(i + 1));
/* Hack to work around a GitHub bug: it sends
ETags, but ignores If-None-Match. So if we get
the expected ETag on a 200 response, then shut
@ -218,8 +218,8 @@ struct curlFileTransfer : public FileTransfer
return 0;
}
} else if (name == "content-encoding")
encoding = trim(string(line, i + 1));
else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes")
encoding = trim(line.substr(i + 1));
else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes")
acceptRanges = true;
}
}
@ -340,7 +340,7 @@ struct curlFileTransfer : public FileTransfer
if (writtenToSink)
curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
result.data = std::make_shared<std::string>();
result.data.clear();
result.bodySize = 0;
}
@ -434,21 +434,21 @@ struct curlFileTransfer : public FileTransfer
attempt++;
std::shared_ptr<std::string> response;
std::optional<std::string> response;
if (errorSink)
response = errorSink->s;
response = std::move(errorSink->s);
auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
? FileTransferError(Interrupted, std::move(response), "%s of '%s' was interrupted", request.verb(), request.uri)
: httpStatus != 0
? FileTransferError(err,
response,
std::move(response),
fmt("unable to %s '%s': HTTP error %d ('%s')",
request.verb(), request.uri, httpStatus, statusMsg)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
)
: FileTransferError(err,
response,
std::move(response),
fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri, curl_easy_strerror(code), code));
@ -704,8 +704,8 @@ struct curlFileTransfer : public FileTransfer
auto s3Res = s3Helper.getObject(bucketName, key);
FileTransferResult res;
if (!s3Res.data)
throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
res.data = s3Res.data;
throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri);
res.data = std::move(*s3Res.data);
callback(std::move(res));
#else
throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
@ -859,25 +859,25 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
}
template<typename... Args>
FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
FileTransferError::FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args)
: Error(args...), error(error), response(response)
{
const auto hf = hintfmt(args...);
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
// to print different messages for different verbosity levels. For now
// we add some heuristics for detecting when we want to show the response.
if (response && (response->size() < 1024 || response->find("<html>") != string::npos))
if (response && (response->size() < 1024 || response->find("<html>") != std::string::npos))
err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response));
else
err.msg = hf;
}
bool isUri(const string & s)
bool isUri(std::string_view s)
{
if (s.compare(0, 8, "channel:") == 0) return true;
size_t pos = s.find("://");
if (pos == string::npos) return false;
string scheme(s, 0, pos);
if (pos == std::string::npos) return false;
std::string scheme(s, 0, pos);
return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
}

12
src/libstore/filetransfer.hh
View file

@ -59,7 +59,7 @@ struct FileTransferRequest
unsigned int baseRetryTimeMs = 250;
ActivityId parentAct;
bool decompress = true;
std::shared_ptr<std::string> data;
std::optional<std::string> data;
std::string mimeType;
std::function<void(std::string_view data)> dataCallback;
@ -77,7 +77,7 @@ struct FileTransferResult
bool cached = false;
std::string etag;
std::string effectiveUri;
std::shared_ptr<std::string> data;
std::string data;
uint64_t bodySize = 0;
};
@ -119,17 +119,17 @@ class FileTransferError : public Error
{
public:
FileTransfer::Error error;
std::shared_ptr<string> response; // intentionally optional
std::optional<std::string> response; // intentionally optional
template<typename... Args>
FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args);
virtual const char* sname() const override { return "FileTransferError"; }
};
bool isUri(const string & s);
bool isUri(std::string_view s);
/* Resolve deprecated 'channel:<foo>' URLs. */
std::string resolveUri(const std::string & uri);
std::string resolveUri(std::string_view uri);
}

23
src/libstore/gc.cc
View file

@ -47,9 +47,8 @@ static void makeSymlink(const Path & link, const Path & target)
void LocalStore::addIndirectRoot(const Path & path)
{
string hash = hashString(htSHA1, path).to_string(Base32, false);
Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
% stateDir % gcRootsDir % hash).str());
std::string hash = hashString(htSHA1, path).to_string(Base32, false);
Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
makeSymlink(realRoot, path);
}
@ -162,7 +161,7 @@ void LocalStore::addTempRoot(const StorePath & path)
}
/* Append the store path to the temporary roots file. */
string s = printStorePath(path) + '\0';
auto s = printStorePath(path) + '\0';
writeFull(state->fdTempRoots.get(), s);
}
@ -203,12 +202,12 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
}
/* Read the entire file. */
string contents = readFile(fd.get());
auto contents = readFile(fd.get());
/* Extract the roots. */
string::size_type pos = 0, end;
std::string::size_type pos = 0, end;
while ((end = contents.find((char) 0, pos)) != string::npos) {
while ((end = contents.find((char) 0, pos)) != std::string::npos) {
Path root(contents, pos, end - pos);
debug("got temporary root '%s'", root);
tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid));
@ -305,7 +304,7 @@ Roots LocalStore::findRoots(bool censor)
typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
static void readProcLink(const string & file, UncheckedRoots & roots)
static void readProcLink(const std::string & file, UncheckedRoots & roots)
{
/* 64 is the starting buffer size gnu readlink uses... */
auto bufsiz = ssize_t{64};
@ -328,7 +327,7 @@ try_again:
.emplace(file);
}
static string quoteRegexChars(const string & raw)
static std::string quoteRegexChars(const std::string & raw)
{
static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
return std::regex_replace(raw, specialRegex, R"(\$&)");
@ -383,7 +382,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
try {
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n");
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex))
@ -784,7 +783,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
string name = dirent->d_name;
std::string name = dirent->d_name;
if (name == "." || name == ".." || name == linksName) continue;
if (auto storePath = maybeParseStorePath(storeDir + "/" + name))
@ -825,7 +824,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
string name = dirent->d_name;
std::string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name;

4
src/libstore/globals.cc
View file

@ -100,7 +100,7 @@ std::vector<Path> getUserConfigFiles()
// Use the paths specified in NIX_USER_CONF_FILES if it has been defined
auto nixConfFiles = getEnv("NIX_USER_CONF_FILES");
if (nixConfFiles.has_value()) {
return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":");
return tokenizeString<std::vector<std::string>>(nixConfFiles.value(), ":");
}
// Use the paths specified by the XDG spec
@ -181,7 +181,7 @@ bool Settings::isWSL1()
return hasSuffix(utsbuf.release, "-Microsoft");
}
const string nixVersion = PACKAGE_VERSION;
const std::string nixVersion = PACKAGE_VERSION;
NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
{SandboxMode::smEnabled, true},

11
src/libstore/globals.hh
View file

@ -113,7 +113,7 @@ public:
bool verboseBuild = true;
Setting<size_t> logLines{this, 10, "log-lines",
"If `verbose-build` is false, the number of lines of the tail of "
"The number of lines of the tail of "
"the log to show if a build fails."};
MaxBuildJobsSetting maxBuildJobs{
@ -966,6 +966,13 @@ public:
Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config",
"Whether to accept nix configuration from a flake without prompting."};
Setting<std::string> commitLockFileSummary{
this, "", "commit-lockfile-summary",
R"(
The commit summary to use when committing changed flake lock files. If
empty, the summary is generated based on the action performed.
)"};
};
@ -981,6 +988,6 @@ void loadConfFile();
// Used by the Settings constructor
std::vector<Path> getUserConfigFiles();
extern const string nixVersion;
extern const std::string nixVersion;
}

8
src/libstore/http-binary-cache-store.cc
View file

@ -126,7 +126,7 @@ protected:
const std::string & mimeType) override
{
auto req = makeRequest(path);
req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
req.data = StreamToSourceAdapter(istream).drain();
req.mimeType = mimeType;
try {
getFileTransfer()->upload(req);
@ -159,7 +159,7 @@ protected:
}
void getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept override
Callback<std::optional<std::string>> callback) noexcept override
{
checkEnabled();
@ -170,10 +170,10 @@ protected:
getFileTransfer()->enqueueFileTransfer(request,
{[callbackPtr, this](std::future<FileTransferResult> result) {
try {
(*callbackPtr)(result.get().data);
(*callbackPtr)(std::move(result.get().data));
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return (*callbackPtr)(std::shared_ptr<std::string>());
return (*callbackPtr)({});
maybeDisable();
callbackPtr->rethrow();
} catch (...) {

24
src/libstore/legacy-ssh-store.cc
View file

@ -48,7 +48,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
static std::set<std::string> uriSchemes() { return {"ssh"}; }
LegacySSHStore(const string & scheme, const string & host, const Params & params)
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params)
, LegacySSHStoreConfig(params)
, Store(params)
@ -94,7 +94,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
conn->sshConn->in.close();
auto msg = conn->from.drain();
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
host, chomp(*saved.s + msg));
host, chomp(saved.s + msg));
}
conn->remoteVersion = readInt(conn->from);
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
@ -107,7 +107,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
return conn;
};
string getUri() override
std::string getUri() override
{
return *uriSchemes().begin() + "://" + host;
}
@ -225,13 +225,21 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair, const StorePathSet & references) override
StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override
{ unsupported("addToStore"); }
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override
{ unsupported("addTextToStore"); }
private:

1
src/libstore/local-binary-cache-store.cc
View file

@ -96,6 +96,7 @@ void LocalBinaryCacheStore::init()
createDirs(binaryCacheDir + "/" + realisationsPrefix);
if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo");
createDirs(binaryCacheDir + "/log");
BinaryCacheStore::init();
}

18
src/libstore/local-fs-store.cc
View file

@ -85,36 +85,34 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
}
const string LocalFSStore::drvsLogDir = "drvs";
const std::string LocalFSStore::drvsLogDir = "drvs";
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
std::optional<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
{
auto path = path_;
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
if (!info->deriver) return nullptr;
if (!info->deriver) return std::nullopt;
path = *info->deriver;
} catch (InvalidPath &) {
return nullptr;
return std::nullopt;
}
}
auto baseName = std::string(baseNameOf(printStorePath(path)));
auto baseName = path.to_string();
for (int j = 0; j < 2; j++) {
Path logPath =
j == 0
? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2))
? fmt("%s/%s/%s/%s", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2))
: fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
Path logBz2Path = logPath + ".bz2";
if (pathExists(logPath))
return std::make_shared<std::string>(readFile(logPath));
return readFile(logPath);
else if (pathExists(logBz2Path)) {
try {
@ -124,7 +122,7 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
}
return nullptr;
return std::nullopt;
}
}

5
src/libstore/local-fs-store.hh
View file

@ -27,7 +27,7 @@ class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
{
public:
const static string drvsLogDir;
const static std::string drvsLogDir;
LocalFSStore(const Params & params);
@ -45,7 +45,8 @@ public:
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
}
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
std::optional<std::string> getBuildLog(const StorePath & path) override;
};
}

68
src/libstore/local-store.cc
View file

@ -9,6 +9,7 @@
#include "callback.hh"
#include "topo-sort.hh"
#include "finally.hh"
#include "compression.hh"
#include <iostream>
#include <algorithm>
@ -69,7 +70,7 @@ int getSchema(Path schemaPath)
{
int curSchema = 0;
if (pathExists(schemaPath)) {
string s = readFile(schemaPath);
auto s = readFile(schemaPath);
auto n = string2Int<int>(s);
if (!n)
throw Error("'%1%' is corrupt", schemaPath);
@ -238,7 +239,7 @@ LocalStore::LocalStore(const Params & params)
res = posix_fallocate(fd.get(), 0, settings.reservedSize);
#endif
if (res == -1) {
writeFull(fd.get(), string(settings.reservedSize, 'X'));
writeFull(fd.get(), std::string(settings.reservedSize, 'X'));
[[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize);
}
}
@ -449,7 +450,7 @@ void LocalStore::openDB(State & state, bool create)
throw SysError("Nix database directory '%1%' is not writable", dbDir);
/* Open the Nix database. */
string dbPath = dbDir + "/db.sqlite";
std::string dbPath = dbDir + "/db.sqlite";
auto & db(state.db);
state.db = SQLite(dbPath, create);
@ -470,19 +471,19 @@ void LocalStore::openDB(State & state, bool create)
should be safe enough. If the user asks for it, don't sync at
all. This can cause database corruption if the system
crashes. */
string syncMode = settings.fsyncMetadata ? "normal" : "off";
std::string syncMode = settings.fsyncMetadata ? "normal" : "off";
db.exec("pragma synchronous = " + syncMode);
/* Set the SQLite journal mode. WAL mode is fastest, so it's the
default. */
string mode = settings.useSQLiteWAL ? "wal" : "truncate";
string prevMode;
std::string mode = settings.useSQLiteWAL ? "wal" : "truncate";
std::string prevMode;
{
SQLiteStmt stmt;
stmt.create(db, "pragma main.journal_mode;");
if (sqlite3_step(stmt) != SQLITE_ROW)
throwSQLiteError(db, "querying journal mode");
prevMode = string((const char *) sqlite3_column_text(stmt, 0));
prevMode = std::string((const char *) sqlite3_column_text(stmt, 0));
}
if (prevMode != mode &&
sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
@ -678,7 +679,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
{
assert(drvPath.isDerivation());
std::string drvName(drvPath.name());
drvName = string(drvName, 0, drvName.size() - drvExtension.size());
drvName = drvName.substr(0, drvName.size() - drvExtension.size());
auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
{
@ -785,7 +786,11 @@ void LocalStore::registerDrvOutput(const Realisation & info)
});
}
void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
void LocalStore::cacheDrvOutputMapping(
State & state,
const uint64_t deriver,
const std::string & outputName,
const StorePath & output)
{
retrySQLite<void>([&]() {
state.stmts->AddDerivationOutput.use()
@ -794,7 +799,6 @@ void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, co
(printStorePath(output))
.exec();
});
}
@ -1307,7 +1311,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
canonicalisePathMetaData(realPath, -1);
optimisePath(realPath); // FIXME: combine with hashPath()
optimisePath(realPath, repair); // FIXME: combine with hashPath()
registerValidPath(info);
}
@ -1317,7 +1321,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
}
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
{
/* For computing the store path. */
@ -1419,7 +1423,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
optimisePath(realPath);
optimisePath(realPath, repair);
ValidPathInfo info { dstPath, narHash.first };
info.narSize = narHash.second;
@ -1435,7 +1439,9 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
}
StorePath LocalStore::addTextToStore(const string & name, const string & s,
StorePath LocalStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references, RepairFlag repair)
{
auto hash = hashString(htSHA256, s);
@ -1461,12 +1467,12 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
StringSink sink;
dumpString(s, sink);
auto narHash = hashString(htSHA256, *sink.s);
auto narHash = hashString(htSHA256, sink.s);
optimisePath(realPath);
optimisePath(realPath, repair);
ValidPathInfo info { dstPath, narHash };
info.narSize = sink.s->size();
info.narSize = sink.s.size();
info.references = references;
info.ca = TextHash { .hash = hash };
registerValidPath(info);
@ -1547,7 +1553,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
for (auto & link : readDirectory(linksDir)) {
printMsg(lvlTalkative, "checking contents of '%s'", link.name);
Path linkPath = linksDir + "/" + link.name;
string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
if (hash != link.name) {
printError("link '%s' was modified! expected hash '%s', got '%s'",
linkPath, link.name, hash);
@ -1898,4 +1904,30 @@ FixedOutputHash LocalStore::hashCAPath(
};
}
void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
assert(drvPath.isDerivation());
auto baseName = drvPath.to_string();
auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
if (pathExists(logPath)) return;
createDirs(dirOf(logPath));
auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
writeFile(tmpFile, compress("bzip2", log));
if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
}
std::optional<std::string> LocalStore::getVersion()
{
return nixVersion;
}
} // namespace nix

28
src/libstore/local-store.hh
View file

@ -144,11 +144,14 @@ public:
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, const string & name,
StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override;
void addTempRoot(const StorePath & path) override;
@ -172,8 +175,9 @@ public:
void optimiseStore() override;
/* Optimise a single store path. */
void optimisePath(const Path & path);
/* Optimise a single store path. Optionally, test the encountered
symlinks for corruption. */
void optimisePath(const Path & path, RepairFlag repair);
bool verifyStore(bool checkContents, RepairFlag repair) override;
@ -203,13 +207,19 @@ public:
derivation 'deriver'. */
void registerDrvOutput(const Realisation & info) override;
void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
void cacheDrvOutputMapping(
State & state,
const uint64_t deriver,
const std::string & outputName,
const StorePath & output);
std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id);
std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id);
void queryRealisationUncached(const DrvOutput&,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
std::optional<std::string> getVersion() override;
private:
int getSchema();
@ -253,7 +263,7 @@ private:
InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, RepairFlag repair);
// Internal versions that are not wrapped in retry_sqlite.
bool isValidPath_(State & state, const StorePath & path);
@ -279,6 +289,8 @@ private:
const std::string_view pathHash
);
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
friend struct LocalDerivationGoal;
friend struct PathSubstitutionGoal;
friend struct SubstitutionGoal;
@ -287,7 +299,7 @@ private:
typedef std::pair<dev_t, ino_t> Inode;
typedef set<Inode> InodesSeen;
typedef std::set<Inode> InodesSeen;
/* "Fix", or canonicalise, the meta-data of the files in a store path

4
src/libstore/lock.hh
View file

@ -13,7 +13,7 @@ private:
AutoCloseFD fdUserLock;
bool isEnabled = false;
string user;
std::string user;
uid_t uid = 0;
gid_t gid = 0;
std::vector<gid_t> supplementaryGIDs;
@ -23,7 +23,7 @@ public:
void kill();
string getUser() { return user; }
std::string getUser() { return user; }
uid_t getUID() { assert(uid); return uid; }
uid_t getGID() { assert(gid); return gid; }
std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }

21
src/libstore/machines.cc
View file

@ -39,19 +39,19 @@ Machine::Machine(decltype(storeUri) storeUri,
sshPublicHostKey(sshPublicHostKey)
{}
bool Machine::allSupported(const std::set<string> & features) const
bool Machine::allSupported(const std::set<std::string> & features) const
{
return std::all_of(features.begin(), features.end(),
[&](const string & feature) {
[&](const std::string & feature) {
return supportedFeatures.count(feature) ||
mandatoryFeatures.count(feature);
});
}
bool Machine::mandatoryMet(const std::set<string> & features) const
bool Machine::mandatoryMet(const std::set<std::string> & features) const
{
return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
[&](const string & feature) {
[&](const std::string & feature) {
return features.count(feature);
});
}
@ -89,7 +89,7 @@ ref<Store> Machine::openStore() const
static std::vector<std::string> expandBuilderLines(const std::string & builders)
{
std::vector<std::string> result;
for (auto line : tokenizeString<std::vector<string>>(builders, "\n;")) {
for (auto line : tokenizeString<std::vector<std::string>>(builders, "\n;")) {
trim(line);
line.erase(std::find(line.begin(), line.end(), '#'), line.end());
if (line.empty()) continue;
@ -117,7 +117,7 @@ static std::vector<std::string> expandBuilderLines(const std::string & builders)
static Machine parseBuilderLine(const std::string & line)
{
const auto tokens = tokenizeString<std::vector<string>>(line);
const auto tokens = tokenizeString<std::vector<std::string>>(line);
auto isSet = [&](size_t fieldIndex) {
return tokens.size() > fieldIndex && tokens[fieldIndex] != "" && tokens[fieldIndex] != "-";
@ -146,17 +146,18 @@ static Machine parseBuilderLine(const std::string & line)
return {
tokens[0],
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
isSet(1) ? tokenizeString<std::vector<std::string>>(tokens[1], ",") : std::vector<std::string>{settings.thisSystem},
isSet(2) ? tokens[2] : "",
isSet(3) ? parseUnsignedIntField(3) : 1U,
isSet(4) ? parseUnsignedIntField(4) : 1U,
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{},
isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{},
isSet(7) ? ensureBase64(7) : ""
};
}
static Machines parseBuilderLines(const std::vector<std::string>& builders) {
static Machines parseBuilderLines(const std::vector<std::string> & builders)
{
Machines result;
std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine);
return result;

14
src/libstore/machines.hh
View file

@ -8,19 +8,19 @@ class Store;
struct Machine {
const string storeUri;
const std::vector<string> systemTypes;
const string sshKey;
const std::string storeUri;
const std::vector<std::string> systemTypes;
const std::string sshKey;
const unsigned int maxJobs;
const unsigned int speedFactor;
const std::set<string> supportedFeatures;
const std::set<string> mandatoryFeatures;
const std::set<std::string> supportedFeatures;
const std::set<std::string> mandatoryFeatures;
const std::string sshPublicHostKey;
bool enabled = true;
bool allSupported(const std::set<string> & features) const;
bool allSupported(const std::set<std::string> & features) const;
bool mandatoryMet(const std::set<string> & features) const;
bool mandatoryMet(const std::set<std::string> & features) const;
Machine(decltype(storeUri) storeUri,
decltype(systemTypes) systemTypes,

24
src/libstore/names.cc
View file

@ -56,8 +56,8 @@ bool DrvName::matches(const DrvName & n)
}
string nextComponent(string::const_iterator & p,
const string::const_iterator end)
std::string_view nextComponent(std::string_view::const_iterator & p,
const std::string_view::const_iterator end)
{
/* Skip any dots and dashes (component separators). */
while (p != end && (*p == '.' || *p == '-')) ++p;
@ -67,18 +67,18 @@ string nextComponent(string::const_iterator & p,
/* If the first character is a digit, consume the longest sequence
of digits. Otherwise, consume the longest sequence of
non-digit, non-separator characters. */
string s;
auto s = p;
if (isdigit(*p))
while (p != end && isdigit(*p)) s += *p++;
while (p != end && isdigit(*p)) p++;
else
while (p != end && (!isdigit(*p) && *p != '.' && *p != '-'))
s += *p++;
p++;
return s;
return {s, size_t(p - s)};
}
static bool componentsLT(const string & c1, const string & c2)
static bool componentsLT(const std::string_view c1, const std::string_view c2)
{
auto n1 = string2Int<int>(c1);
auto n2 = string2Int<int>(c2);
@ -94,14 +94,14 @@ static bool componentsLT(const string & c1, const string & c2)
}
int compareVersions(const string & v1, const string & v2)
int compareVersions(const std::string_view v1, const std::string_view v2)
{
string::const_iterator p1 = v1.begin();
string::const_iterator p2 = v2.begin();
auto p1 = v1.begin();
auto p2 = v2.begin();
while (p1 != v1.end() || p2 != v2.end()) {
string c1 = nextComponent(p1, v1.end());
string c2 = nextComponent(p2, v2.end());
auto c1 = nextComponent(p1, v1.end());
auto c2 = nextComponent(p2, v2.end());
if (componentsLT(c1, c2)) return -1;
else if (componentsLT(c2, c1)) return 1;
}

14
src/libstore/names.hh
View file

@ -10,9 +10,9 @@ struct Regex;
struct DrvName
{
string fullName;
string name;
string version;
std::string fullName;
std::string name;
std::string version;
unsigned int hits;
DrvName();
@ -25,11 +25,11 @@ private:
std::unique_ptr<Regex> regex;
};
typedef list<DrvName> DrvNames;
typedef std::list<DrvName> DrvNames;
string nextComponent(string::const_iterator & p,
const string::const_iterator end);
int compareVersions(const string & v1, const string & v2);
std::string_view nextComponent(std::string_view::const_iterator & p,
const std::string_view::const_iterator end);
int compareVersions(const std::string_view v1, const std::string_view v2);
DrvNames drvNamesFromArgs(const Strings & opArgs);
}

10
src/libstore/nar-accessor.cc
View file

@ -28,7 +28,7 @@ struct NarMember
struct NarAccessor : public FSAccessor
{
std::shared_ptr<const std::string> nar;
std::optional<const std::string> nar;
GetNarBytes getNarBytes;
@ -90,7 +90,7 @@ struct NarAccessor : public FSAccessor
void receiveContents(std::string_view data) override
{ }
void createSymlink(const Path & path, const string & target) override
void createSymlink(const Path & path, const std::string & target) override
{
createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
@ -104,7 +104,7 @@ struct NarAccessor : public FSAccessor
}
};
NarAccessor(ref<const std::string> nar) : nar(nar)
NarAccessor(std::string && _nar) : nar(_nar)
{
StringSource source(*nar);
NarIndexer indexer(*this, source);
@ -224,9 +224,9 @@ struct NarAccessor : public FSAccessor
}
};
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
ref<FSAccessor> makeNarAccessor(std::string && nar)
{
return make_ref<NarAccessor>(nar);
return make_ref<NarAccessor>(std::move(nar));
}
ref<FSAccessor> makeNarAccessor(Source & source)

2
src/libstore/nar-accessor.hh
View file

@ -10,7 +10,7 @@ struct Source;
/* Return an object that provides access to the contents of a NAR
file. */
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
ref<FSAccessor> makeNarAccessor(std::string && nar);
ref<FSAccessor> makeNarAccessor(Source & source);

2
src/libstore/nar-info.cc
View file

@ -11,7 +11,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
return Error("NAR info file '%1%' is corrupt", whence);
};
auto parseHashField = [&](const string & s) {
auto parseHashField = [&](const std::string & s) {
try {
return Hash::parseAnyPrefixed(s);
} catch (BadHash &) {

42
src/libstore/optimise-store.cc
View file

@ -26,7 +26,7 @@ static void makeWritable(const Path & path)
struct MakeReadOnly
{
Path path;
MakeReadOnly(const Path & path) : path(path) { }
MakeReadOnly(const PathView path) : path(path) { }
~MakeReadOnly()
{
try {
@ -77,7 +77,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
continue;
}
string name = dirent->d_name;
std::string name = dirent->d_name;
if (name == "." || name == "..") continue;
names.push_back(name);
}
@ -88,7 +88,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
const Path & path, InodeHash & inodeHash)
const Path & path, InodeHash & inodeHash, RepairFlag repair)
{
checkInterrupt();
@ -110,7 +110,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
for (auto & i : names)
optimisePath_(act, stats, path + "/" + i, inodeHash);
optimisePath_(act, stats, path + "/" + i, inodeHash, repair);
return;
}
@ -151,7 +151,20 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* Check if this is a known hash. */
Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
retry:
/* Maybe delete the link, if it has been corrupted. */
if (pathExists(linkPath)) {
auto stLink = lstat(linkPath);
if (st.st_size != stLink.st_size
|| (repair && hash != hashPath(htSHA256, linkPath).first))
{
// XXX: Consider overwriting linkPath with our valid version.
warn("removing corrupted link '%s'", linkPath);
warn("There may be more corrupted paths."
"\nYou should run `nix-store --verify --check-contents --repair` to fix them all");
unlink(linkPath.c_str());
}
}
if (!pathExists(linkPath)) {
/* Nope, create a hard link in the links directory. */
if (link(path.c_str(), linkPath.c_str()) == 0) {
@ -187,23 +200,18 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
return;
}
if (st.st_size != stLink.st_size) {
warn("removing corrupted link '%s'", linkPath);
unlink(linkPath.c_str());
goto retry;
}
printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath);
/* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its
permissions). */
bool mustToggle = dirOf(path) != realStoreDir.get();
if (mustToggle) makeWritable(dirOf(path));
const Path dirOfPath(dirOf(path));
bool mustToggle = dirOfPath != realStoreDir.get();
if (mustToggle) makeWritable(dirOfPath);
/* When we're done, make the directory read-only again and reset
its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : "");
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
% realStoreDir % getpid() % random()).str();
@ -260,7 +268,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i)));
optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash);
optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair);
}
done++;
act.progress(done, paths.size());
@ -278,12 +286,12 @@ void LocalStore::optimiseStore()
stats.filesLinked);
}
void LocalStore::optimisePath(const Path & path)
void LocalStore::optimisePath(const Path & path, RepairFlag repair)
{
OptimiseStats stats;
InodeHash inodeHash;
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash);
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash, repair);
}

2
src/libstore/parsed-derivations.cc
View file

@ -170,7 +170,7 @@ std::string writeStructuredAttrsShell(const nlohmann::json & json)
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
if (value.is_string())
return shellEscape(value);
return shellEscape(value.get<std::string_view>());
if (value.is_number()) {
auto f = value.get<float>();

4
src/libstore/path-with-outputs.cc
View file

@ -49,9 +49,9 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? std::make_pair(s, std::set<string>())
? std::make_pair(s, std::set<std::string>())
: std::make_pair(((std::string_view) s).substr(0, n),
tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
tokenizeString<std::set<std::string>>(((std::string_view) s).substr(n + 1), ","));
}

2
src/libstore/path.hh
View file

@ -62,7 +62,7 @@ public:
typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap;
typedef std::map<std::string, StorePath> OutputPathMap;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;

4
src/libstore/pathlocks.cc
View file

@ -74,7 +74,7 @@ PathLocks::PathLocks()
}
PathLocks::PathLocks(const PathSet & paths, const string & waitMsg)
PathLocks::PathLocks(const PathSet & paths, const std::string & waitMsg)
: deletePaths(false)
{
lockPaths(paths, waitMsg);
@ -82,7 +82,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg)
bool PathLocks::lockPaths(const PathSet & paths,
const string & waitMsg, bool wait)
const std::string & waitMsg, bool wait)
{
assert(fds.empty());

6
src/libstore/pathlocks.hh
View file

@ -20,15 +20,15 @@ class PathLocks
{
private:
typedef std::pair<int, Path> FDPair;
list<FDPair> fds;
std::list<FDPair> fds;
bool deletePaths;
public:
PathLocks();
PathLocks(const PathSet & paths,
const string & waitMsg = "");
const std::string & waitMsg = "");
bool lockPaths(const PathSet & _paths,
const string & waitMsg = "",
const std::string & waitMsg = "",
bool wait = true);
~PathLocks();
void unlock();

16
src/libstore/profiles.cc
View file

@ -15,12 +15,12 @@ namespace nix {
/* Parse a generation name of the format
`<profilename>-<number>-link'. */
static std::optional<GenerationNumber> parseName(const string & profileName, const string & name)
static std::optional<GenerationNumber> parseName(const std::string & profileName, const std::string & name)
{
if (string(name, 0, profileName.size() + 1) != profileName + "-") return {};
string s = string(name, profileName.size() + 1);
string::size_type p = s.find("-link");
if (p == string::npos) return {};
if (name.substr(0, profileName.size() + 1) != profileName + "-") return {};
auto s = name.substr(profileName.size() + 1);
auto p = s.find("-link");
if (p == std::string::npos) return {};
if (auto n = string2Int<unsigned int>(s.substr(0, p)))
return *n;
else
@ -209,13 +209,13 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
}
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun)
void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun)
{
if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd')
throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec);
time_t curTime = time(0);
string strDays = string(timeSpec, 0, timeSpec.size() - 1);
auto strDays = timeSpec.substr(0, timeSpec.size() - 1);
auto days = string2Int<int>(strDays);
if (!days || *days < 1)
@ -274,7 +274,7 @@ void lockProfile(PathLocks & lock, const Path & profile)
}
string optimisticLockProfile(const Path & profile)
std::string optimisticLockProfile(const Path & profile)
{
return pathExists(profile) ? readLink(profile) : "";
}

4
src/libstore/profiles.hh
View file

@ -42,7 +42,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun);
void switchLink(Path link, Path target);
@ -66,7 +66,7 @@ void lockProfile(PathLocks & lock, const Path & profile);
generally cheap, since the build results are still in the Nix
store. Most of the time, only the user environment has to be
rebuilt. */
string optimisticLockProfile(const Path & profile);
std::string optimisticLockProfile(const Path & profile);
/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create
it. */

2
src/libstore/realisation.cc
View file

@ -78,7 +78,7 @@ Realisation Realisation::fromJSON(
auto fieldIterator = json.find(fieldName);
if (fieldIterator == json.end())
return std::nullopt;
return *fieldIterator;
return {*fieldIterator};
};
auto getField = [&](std::string fieldName) -> std::string {
if (auto field = getOptionalField(fieldName))

4
src/libstore/references.cc
View file

@ -68,7 +68,7 @@ void RefScanSink::operator () (std::string_view data)
std::pair<StorePathSet, HashResult> scanForReferences(
const string & path,
const std::string & path,
const StorePathSet & refs)
{
HashSink hashSink { htSHA256 };
@ -121,7 +121,7 @@ void RewritingSink::operator () (std::string_view data)
s.append(data);
size_t j = 0;
while ((j = s.find(from, j)) != string::npos) {
while ((j = s.find(from, j)) != std::string::npos) {
matches.push_back(pos + j);
s.replace(j, from.size(), to);
}

30
src/libstore/remote-fs-accessor.cc
View file

@ -22,9 +22,18 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin
return fmt("%s/%s.%s", cacheDir, hashPart, ext);
}
void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor)
ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
{
if (cacheDir != "") {
try {
/* FIXME: do this asynchronously. */
writeFile(makeCacheFile(hashPart, "nar"), nar);
} catch (...) {
ignoreException();
}
}
auto narAccessor = makeNarAccessor(std::move(nar));
nars.emplace(hashPart, narAccessor);
if (cacheDir != "") {
@ -33,14 +42,12 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string &
JSONPlaceholder jsonRoot(str);
listNar(jsonRoot, narAccessor, "", true);
writeFile(makeCacheFile(hashPart, "ls"), str.str());
/* FIXME: do this asynchronously. */
writeFile(makeCacheFile(hashPart, "nar"), nar);
} catch (...) {
ignoreException();
}
}
return narAccessor;
}
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath)
@ -55,7 +62,6 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
auto i = nars.find(std::string(storePath.hashPart()));
if (i != nars.end()) return {i->second, restPath};
StringSink sink;
std::string listing;
Path cacheFile;
@ -86,19 +92,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
} catch (SysError &) { }
try {
*sink.s = nix::readFile(cacheFile);
auto narAccessor = makeNarAccessor(sink.s);
auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
}
StringSink sink;
store->narFromPath(storePath, sink);
auto narAccessor = makeNarAccessor(sink.s);
addToCache(storePath.hashPart(), *sink.s, narAccessor);
return {narAccessor, restPath};
return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath};
}
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)

3
src/libstore/remote-fs-accessor.hh
View file

@ -20,8 +20,7 @@ class RemoteFSAccessor : public FSAccessor
Path makeCacheFile(std::string_view hashPart, const std::string & ext);
void addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor);
ref<FSAccessor> addToCache(std::string_view hashPart, std::string && nar);
public:

43
src/libstore/remote-store.cc
View file

@ -172,7 +172,7 @@ void RemoteStore::initConnection(Connection & conn)
it. */
conn.closeWrite();
auto msg = conn.from.drain();
throw Error("protocol mismatch, got '%s'", chomp(*saved.s + msg));
throw Error("protocol mismatch, got '%s'", chomp(saved.s + msg));
}
conn.from >> conn.daemonVersion;
@ -188,7 +188,12 @@ void RemoteStore::initConnection(Connection & conn)
}
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
conn.to << false;
conn.to << false; // obsolete reserveSpace
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 33) {
conn.to.flush();
conn.daemonNixVersion = readString(conn.from);
}
auto ex = conn.processStderr();
if (ex) std::rethrow_exception(ex);
@ -495,7 +500,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
ref<const ValidPathInfo> RemoteStore::addCAToStore(
Source & dump,
const string & name,
std::string_view name,
ContentAddressMethod caMethod,
const StorePathSet & references,
RepairFlag repair)
@ -577,7 +582,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
}
StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
{
return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
@ -656,8 +661,11 @@ void RemoteStore::addMultipleToStore(
}
StorePath RemoteStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
StorePath RemoteStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair)
{
StringSource source(s);
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
@ -908,6 +916,25 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
}
void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
auto conn(getConnection());
conn->to << wopAddBuildLog << drvPath.to_string();
StringSource source(log);
conn.withFramedSink([&](Sink & sink) {
source.drainInto(sink);
});
readInt(conn->from);
}
std::optional<std::string> RemoteStore::getVersion()
{
auto conn(getConnection());
return conn->daemonNixVersion;
}
void RemoteStore::connect()
{
auto conn(getConnection());
@ -976,7 +1003,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
auto msg = readNum<uint64_t>(from);
if (msg == STDERR_WRITE) {
string s = readString(from);
auto s = readString(from);
if (!sink) throw Error("no sink");
(*sink)(s);
}
@ -993,7 +1020,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
return std::make_exception_ptr(readError(from));
} else {
string error = readString(from);
auto error = readString(from);
unsigned int status = readInt(from);
return std::make_exception_ptr(Error(status, error));
}

16
src/libstore/remote-store.hh
View file

@ -66,13 +66,13 @@ public:
/* Add a content-addressable store path. `dump` will be drained. */
ref<const ValidPathInfo> addCAToStore(
Source & dump,
const string & name,
std::string_view name,
ContentAddressMethod caMethod,
const StorePathSet & references,
RepairFlag repair);
/* Add a content-addressable store path. Does not support references. `dump` will be drained. */
StorePath addToStoreFromDump(Source & dump, const string & name,
StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
void addToStore(const ValidPathInfo & info, Source & nar,
@ -83,8 +83,11 @@ public:
RepairFlag repair,
CheckSigsFlag checkSigs) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
@ -116,6 +119,10 @@ public:
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
std::optional<std::string> getVersion() override;
void connect() override;
unsigned int getProtocol() override;
@ -127,6 +134,7 @@ public:
FdSink to;
FdSource from;
unsigned int daemonVersion;
std::optional<std::string> daemonNixVersion;
std::chrono::time_point<std::chrono::steady_clock> startTime;
virtual ~Connection();

13
src/libstore/s3-binary-cache-store.cc
View file

@ -87,7 +87,11 @@ static void initAWS()
});
}
S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint)
S3Helper::S3Helper(
const std::string & profile,
const std::string & region,
const std::string & scheme,
const std::string & endpoint)
: config(makeConfig(region, scheme, endpoint))
, client(make_ref<Aws::S3::S3Client>(
profile == ""
@ -121,7 +125,10 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
}
};
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint)
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
const std::string & region,
const std::string & scheme,
const std::string & endpoint)
{
initAWS();
auto res = make_ref<Aws::Client::ClientConfiguration>();
@ -385,7 +392,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
auto compress = [&](std::string compression)
{
auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
return std::make_shared<std::stringstream>(std::move(*compressed));
return std::make_shared<std::stringstream>(std::move(compressed));
};
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))

4
src/libstore/s3.hh
View file

@ -4,6 +4,8 @@
#include "ref.hh"
#include <optional>
namespace Aws { namespace Client { class ClientConfiguration; } }
namespace Aws { namespace S3 { class S3Client; } }
@ -20,7 +22,7 @@ struct S3Helper
struct FileTransferResult
{
std::shared_ptr<std::string> data;
std::optional<std::string> data;
unsigned int durationMs;
};

2
src/libstore/sqlite.cc
View file

@ -71,7 +71,7 @@ uint64_t SQLite::getLastInsertedRowId()
return sqlite3_last_insert_rowid(db);
}
void SQLiteStmt::create(sqlite3 * db, const string & sql)
void SQLiteStmt::create(sqlite3 * db, const std::string & sql)
{
checkInterrupt();
assert(!stmt);

2
src/libstore/ssh.cc
View file

@ -29,7 +29,7 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
if (!sshPublicHostKey.empty()) {
Path fileName = (Path) *state->tmpDir + "/host-key";
auto p = host.rfind("@");
string thost = p != string::npos ? string(host, p + 1) : host;
std::string thost = p != std::string::npos ? std::string(host, p + 1) : host;
writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n");
args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName});
}

50
src/libstore/store-api.cc
View file

@ -39,7 +39,7 @@ Path Store::followLinksToStore(std::string_view _path) const
Path path = absPath(std::string(_path));
while (!isInStore(path)) {
if (!isLink(path)) break;
string target = readLink(path);
auto target = readLink(path);
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
@ -138,8 +138,8 @@ StorePath Store::makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = std::string { type } + ":" + std::string { hash }
+ ":" + storeDir + ":" + std::string { name };
auto s = std::string(type) + ":" + std::string(hash)
+ ":" + storeDir + ":" + std::string(name);
auto h = compressHash(hashString(htSHA256, s), 20);
return StorePath(h, name);
}
@ -161,7 +161,7 @@ StorePath Store::makeOutputPath(std::string_view id,
static std::string makeType(
const Store & store,
string && type,
std::string && type,
const StorePathSet & references,
bool hasSelfReference = false)
{
@ -229,15 +229,23 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
}
StorePath Store::computeStorePathForText(const string & name, const string & s,
StorePath Store::computeStorePathForText(
std::string_view name,
std::string_view s,
const StorePathSet & references) const
{
return makeTextPath(name, hashString(htSHA256, s), references);
}
StorePath Store::addToStore(const string & name, const Path & _srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references)
StorePath Store::addToStore(
std::string_view name,
const Path & _srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references)
{
Path srcPath(absPath(_srcPath));
auto source = sinkToSource([&](Sink & sink) {
@ -688,10 +696,10 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
/* Return a string accepted by decodeValidPathInfo() that
registers the specified paths as valid. Note: it's the
responsibility of the caller to provide a closure. */
string Store::makeValidityRegistration(const StorePathSet & paths,
std::string Store::makeValidityRegistration(const StorePathSet & paths,
bool showDerivers, bool showHash)
{
string s = "";
std::string s = "";
for (auto & i : paths) {
s += printStorePath(i) + "\n";
@ -761,11 +769,11 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
for (auto & storePath : storePaths) {
auto jsonPath = jsonList.object();
jsonPath.attr("path", printStorePath(storePath));
try {
auto info = queryPathInfo(storePath);
jsonPath.attr("path", printStorePath(info->path));
jsonPath
.attr("narHash", info->narHash.to_string(hashBase, true))
.attr("narSize", info->narSize);
@ -819,6 +827,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
}
} catch (InvalidPath &) {
jsonPath.attr("path", printStorePath(storePath));
jsonPath.attr("valid", false);
}
}
@ -1109,13 +1118,28 @@ void copyClosure(
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
}
void copyClosure(
Store & srcStore,
Store & dstStore,
const StorePathSet & storePaths,
RepairFlag repair,
CheckSigsFlag checkSigs,
SubstituteFlag substitute)
{
if (&srcStore == &dstStore) return;
StorePathSet closure;
srcStore.computeFSClosure(storePaths, closure);
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
}
std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
{
std::string path;
getline(str, path);
if (str.eof()) { return {}; }
if (!hashGiven) {
string s;
std::string s;
getline(str, s);
auto narHash = Hash::parseAny(s, htSHA256);
getline(str, s);
@ -1128,7 +1152,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
std::string deriver;
getline(str, deriver);
if (deriver != "") info.deriver = store.parseStorePath(deriver);
string s;
std::string s;
getline(str, s);
auto n = string2Int<int>(s);
if (!n) throw Error("number expected");
@ -1152,7 +1176,7 @@ std::string Store::showPaths(const StorePathSet & paths)
}
string showPaths(const PathSet & paths)
std::string showPaths(const PathSet & paths)
{
return concatStringsSep(", ", quoteStrings(paths));
}

45
src/libstore/store-api.hh
View file

@ -352,7 +352,9 @@ public:
simply yield a different store path, so other users wouldn't be
affected), but it has some backwards compatibility issues (the
hashing scheme changes), so I'm not doing that for now. */
StorePath computeStorePathForText(const string & name, const string & s,
StorePath computeStorePathForText(
std::string_view name,
std::string_view s,
const StorePathSet & references) const;
/* Check whether a path is valid. */
@ -482,9 +484,14 @@ public:
validity the resulting path. The resulting path is returned.
The function object `filter' can be used to exclude files (see
libutil/archive.hh). */
virtual StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet());
virtual StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive,
HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter,
RepairFlag repair = NoRepair,
const StorePathSet & references = StorePathSet());
/* Copy the contents of a path to the store and register the
validity the resulting path, using a constant amount of
@ -499,15 +506,18 @@ public:
false).
`dump` may be drained */
// FIXME: remove?
virtual StorePath addToStoreFromDump(Source & dump, const string & name,
virtual StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair,
const StorePathSet & references = StorePathSet())
{ unsupported("addToStoreFromDump"); }
/* Like addToStore, but the contents written to the output path is
a regular file containing the given string. */
virtual StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
virtual StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair = NoRepair) = 0;
/**
* Add a mapping indicating that `deriver!outputName` maps to the output path
@ -608,7 +618,7 @@ public:
/* Return a string representing information about the path that
can be loaded into the database using `nix-store --load-db' or
`nix-store --register-validity'. */
string makeValidityRegistration(const StorePathSet & paths,
std::string makeValidityRegistration(const StorePathSet & paths,
bool showDerivers, bool showHash);
/* Write a JSON representation of store path metadata, such as the
@ -724,8 +734,11 @@ public:
/* Return the build log of the specified store path, if available,
or null otherwise. */
virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path)
{ return nullptr; }
virtual std::optional<std::string> getBuildLog(const StorePath & path)
{ return std::nullopt; }
virtual void addBuildLog(const StorePath & path, std::string_view log)
{ unsupported("addBuildLog"); }
/* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */
@ -762,6 +775,9 @@ public:
* (a no-op when there’s no daemon)
*/
virtual void setOptions() { }
virtual std::optional<std::string> getVersion() { return {}; }
protected:
Stats stats;
@ -812,6 +828,13 @@ void copyClosure(
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
void copyClosure(
Store & srcStore, Store & dstStore,
const StorePathSet & paths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
/* Remove the temporary roots file for this process. Any temporary
root becomes garbage after this point unless it has been registered
as a (permanent) root. */
@ -897,7 +920,7 @@ struct RegisterStoreImplementation
/* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */
string showPaths(const PathSet & paths);
std::string showPaths(const PathSet & paths);
std::optional<ValidPathInfo> decodeValidPathInfo(

3
src/libstore/worker-protocol.hh
View file

@ -9,7 +9,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION (1 << 8 | 32)
#define PROTOCOL_VERSION (1 << 8 | 33)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -56,6 +56,7 @@ typedef enum {
wopRegisterDrvOutput = 42,
wopQueryRealisation = 43,
wopAddMultipleToStore = 44,
wopAddBuildLog = 45,
} WorkerOp;