nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-11 21:16:02 +01:00
Code Activity

Merge remote-tracking branch 'upstream/master' into auto-uid-allocation

Browse source
This commit is contained in:
John Ericson 2022-02-28 23:54:20 +00:00
commit dc92b01885
417 changed files with 22620 additions and 35795 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/libutil/abstract-setting-to-json.hh
View file

@ -10,6 +10,7 @@ std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject()
auto obj = AbstractSetting::toJSONObject();
obj.emplace("value", value);
obj.emplace("defaultValue", defaultValue);
obj.emplace("documentDefault", documentDefault);
return obj;
}
}

70
src/libutil/affinity.cc
View file

@ -1,70 +0,0 @@
#include "types.hh"
#include "util.hh"
#include "affinity.hh"
#if __linux__
#include <sched.h>
#endif
namespace nix {
#if __linux__
static bool didSaveAffinity = false;
static cpu_set_t savedAffinity;
std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset)
{
auto count = CPU_COUNT(&cset);
for (int i=0; i < count; ++i)
{
os << (CPU_ISSET(i,&cset) ? "1" : "0");
}
return os;
}
#endif
void setAffinityTo(int cpu)
{
#if __linux__
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
didSaveAffinity = true;
debug(format("locking this thread to CPU %1%") % cpu);
cpu_set_t newAffinity;
CPU_ZERO(&newAffinity);
CPU_SET(cpu, &newAffinity);
if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
printError("failed to lock thread to CPU %1%", cpu);
#endif
}
int lockToCurrentCPU()
{
#if __linux__
int cpu = sched_getcpu();
if (cpu != -1) setAffinityTo(cpu);
return cpu;
#else
return -1;
#endif
}
void restoreAffinity()
{
#if __linux__
if (!didSaveAffinity) return;
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
{
std::ostringstream oss;
oss << savedAffinity;
printError("failed to restore CPU affinity %1%", oss.str());
}
#endif
}
}

9
src/libutil/affinity.hh
View file

@ -1,9 +0,0 @@
#pragma once
namespace nix {
void setAffinityTo(int cpu);
int lockToCurrentCPU();
void restoreAffinity();
}

37
src/libutil/archive.cc
View file

@ -37,12 +37,12 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings);
const std::string narVersionMagic1 = "nix-archive-1";
static string caseHackSuffix = "~nix~case~hack~";
static std::string caseHackSuffix = "~nix~case~hack~";
PathFilter defaultPathFilter = [](const Path &) { return true; };
static void dumpContents(const Path & path, size_t size,
static void dumpContents(const Path & path, off_t size,
Sink & sink)
{
sink << "contents" << size;
@ -76,7 +76,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
sink << "type" << "regular";
if (st.st_mode & S_IXUSR)
sink << "executable" << "";
dumpContents(path, (size_t) st.st_size, sink);
dumpContents(path, st.st_size, sink);
}
else if (S_ISDIR(st.st_mode)) {
@ -84,22 +84,21 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
/* If we're on a case-insensitive system like macOS, undo
the case hack applied by restorePath(). */
std::map<string, string> unhacked;
std::map<std::string, std::string> unhacked;
for (auto & i : readDirectory(path))
if (archiveSettings.useCaseHack) {
string name(i.name);
std::string name(i.name);
size_t pos = i.name.find(caseHackSuffix);
if (pos != string::npos) {
if (pos != std::string::npos) {
debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name));
name.erase(pos);
}
if (unhacked.find(name) != unhacked.end())
if (!unhacked.emplace(name, i.name).second)
throw Error("file name collision in between '%1%' and '%2%'",
(path + "/" + unhacked[name]),
(path + "/" + i.name));
unhacked[name] = i.name;
} else
unhacked[i.name] = i.name;
unhacked.emplace(i.name, i.name);
for (auto & i : unhacked)
if (filter(path + "/" + i.first)) {
@ -125,13 +124,13 @@ void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
}
void dumpString(const std::string & s, Sink & sink)
void dumpString(std::string_view s, Sink & sink)
{
sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")";
}
static SerialisationError badArchive(string s)
static SerialisationError badArchive(const std::string & s)
{
return SerialisationError("bad archive: " + s);
}
@ -172,7 +171,7 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
struct CaseInsensitiveCompare
{
bool operator() (const string & a, const string & b) const
bool operator() (const std::string & a, const std::string & b) const
{
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
@ -181,7 +180,7 @@ struct CaseInsensitiveCompare
static void parse(ParseSink & sink, Source & source, const Path & path)
{
string s;
std::string s;
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
@ -202,7 +201,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
else if (s == "type") {
if (type != tpUnknown)
throw badArchive("multiple type fields");
string t = readString(source);
std::string t = readString(source);
if (t == "regular") {
type = tpRegular;
@ -233,7 +232,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
}
else if (s == "entry" && type == tpDirectory) {
string name, prevName;
std::string name, prevName;
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
@ -247,7 +246,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
break;
} else if (s == "name") {
name = readString(source);
if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos)
throw Error("NAR contains invalid file name '%1%'", name);
if (name <= prevName)
throw Error("NAR directory is not sorted");
@ -270,7 +269,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
}
else if (s == "target" && type == tpSymlink) {
string target = readString(source);
std::string target = readString(source);
sink.createSymlink(path, target);
}
@ -282,7 +281,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
void parseDump(ParseSink & sink, Source & source)
{
string version;
std::string version;
try {
version = readString(source, narVersionMagic1.size());
} catch (SerialisationError & e) {
@ -346,7 +345,7 @@ struct RestoreSink : ParseSink
writeFull(fd.get(), data);
}
void createSymlink(const Path & path, const string & target) override
void createSymlink(const Path & path, const std::string & target) override
{
Path p = dstPath + path;
nix::createSymlink(target, p);

6
src/libutil/archive.hh
View file

@ -48,7 +48,7 @@ namespace nix {
void dumpPath(const Path & path, Sink & sink,
PathFilter & filter = defaultPathFilter);
void dumpString(const std::string & s, Sink & sink);
void dumpString(std::string_view s, Sink & sink);
/* FIXME: fix this API, it sucks. */
struct ParseSink
@ -60,7 +60,7 @@ struct ParseSink
virtual void preallocateContents(uint64_t size) { };
virtual void receiveContents(std::string_view data) { };
virtual void createSymlink(const Path & path, const string & target) { };
virtual void createSymlink(const Path & path, const std::string & target) { };
};
/* If the NAR archive contains a single file at top-level, then save
@ -82,7 +82,7 @@ struct RetrieveRegularNARSink : ParseSink
sink(data);
}
void createSymlink(const Path & path, const string & target) override
void createSymlink(const Path & path, const std::string & target) override
{
regular = false;
}

16
src/libutil/args.cc
View file

@ -39,7 +39,7 @@ void Completions::add(std::string completion, std::string description)
bool Completion::operator<(const Completion & other) const
{ return completion < other.completion || (completion == other.completion && description < other.description); }
bool pathCompletions = false;
CompletionType completionType = ctNormal;
std::shared_ptr<Completions> completions;
std::string completionMarker = "___COMPLETE___";
@ -76,13 +76,13 @@ void Args::parseCmdline(const Strings & _cmdline)
/* Expand compound dash options (i.e., `-qlf' -> `-q -l -f',
`-j3` -> `-j 3`). */
if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) {
*pos = (string) "-" + arg[1];
*pos = (std::string) "-" + arg[1];
auto next = pos; ++next;
for (unsigned int j = 2; j < arg.length(); j++)
if (isalpha(arg[j]))
cmdline.insert(next, (string) "-" + arg[j]);
cmdline.insert(next, (std::string) "-" + arg[j]);
else {
cmdline.insert(next, string(arg, j));
cmdline.insert(next, std::string(arg, j));
break;
}
arg = *pos;
@ -139,7 +139,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
return true;
};
if (string(*pos, 0, 2) == "--") {
if (std::string(*pos, 0, 2) == "--") {
if (auto prefix = needsCompletion(*pos)) {
for (auto & [name, flag] : longFlags) {
if (!hiddenCategories.count(flag->category)
@ -147,12 +147,12 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
completions->add("--" + name, flag->description);
}
}
auto i = longFlags.find(string(*pos, 2));
auto i = longFlags.find(std::string(*pos, 2));
if (i == longFlags.end()) return false;
return process("--" + i->first, *i->second);
}
if (string(*pos, 0, 1) == "-" && pos->size() == 2) {
if (std::string(*pos, 0, 1) == "-" && pos->size() == 2) {
auto c = (*pos)[1];
auto i = shortFlags.find(c);
if (i == shortFlags.end()) return false;
@ -277,7 +277,7 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<
static void _completePath(std::string_view prefix, bool onlyDirs)
{
pathCompletions = true;
completionType = ctFilenames;
glob_t globbuf;
int flags = GLOB_NOESCAPE | GLOB_TILDE;
#ifdef GLOB_ONLYDIR

10
src/libutil/args.hh
View file

@ -158,7 +158,7 @@ public:
}
/* Expect a string argument. */
void expectArg(const std::string & label, string * dest, bool optional = false)
void expectArg(const std::string & label, std::string * dest, bool optional = false)
{
expectArgs({
.label = label,
@ -237,7 +237,13 @@ public:
void add(std::string completion, std::string description = "");
};
extern std::shared_ptr<Completions> completions;
extern bool pathCompletions;
enum CompletionType {
ctNormal,
ctFilenames,
ctAttrs
};
extern CompletionType completionType;
std::optional<std::string> needsCompletion(std::string_view s);

30
src/libutil/compression.cc
View file

@ -16,6 +16,8 @@
namespace nix {
static const int COMPRESSION_LEVEL_DEFAULT = -1;
// Don't feed brotli too much at once.
struct ChunkedCompressionSink : CompressionSink
{
@ -65,14 +67,16 @@ struct ArchiveCompressionSink : CompressionSink
Sink & nextSink;
struct archive * archive;
ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) {
ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink)
{
archive = archive_write_new();
if (!archive) throw Error("failed to initialize libarchive");
check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
check(archive_write_set_format_raw(archive));
if (format == "xz" && parallel) {
if (parallel)
check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
}
if (level != COMPRESSION_LEVEL_DEFAULT)
check(archive_write_set_filter_option(archive, format.c_str(), "compression-level", std::to_string(level).c_str()));
// disable internal buffering
check(archive_write_set_bytes_per_block(archive, 0));
// disable output padding
@ -126,7 +130,11 @@ private:
struct NoneSink : CompressionSink
{
Sink & nextSink;
NoneSink(Sink & nextSink) : nextSink(nextSink) { }
NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink)
{
if (level != COMPRESSION_LEVEL_DEFAULT)
warn("requested compression level '%d' not supported by compression method 'none'", level);
}
void finish() override { flush(); }
void write(std::string_view data) override { nextSink(data); }
};
@ -182,13 +190,13 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
}
};
ref<std::string> decompress(const std::string & method, const std::string & in)
std::string decompress(const std::string & method, std::string_view in)
{
StringSink ssink;
auto sink = makeDecompressionSink(method, ssink);
(*sink)(in);
sink->finish();
return ssink.s;
return std::move(ssink.s);
}
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
@ -257,13 +265,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink
}
};
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
{
std::vector<std::string> la_supports = {
"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"
};
if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel);
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
}
if (method == "none")
return make_ref<NoneSink>(nextSink);
@ -273,13 +281,13 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
throw UnknownCompressionMethod("unknown compression method '%s'", method);
}
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
std::string compress(const std::string & method, std::string_view in, const bool parallel, int level)
{
StringSink ssink;
auto sink = makeCompressionSink(method, ssink, parallel);
auto sink = makeCompressionSink(method, ssink, parallel, level);
(*sink)(in);
sink->finish();
return ssink.s;
return std::move(ssink.s);
}
}

6
src/libutil/compression.hh
View file

@ -15,13 +15,13 @@ struct CompressionSink : BufferedSink, FinishSink
using FinishSink::finish;
};
ref<std::string> decompress(const std::string & method, const std::string & in);
std::string decompress(const std::string & method, std::string_view in);
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1);
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
MakeError(UnknownCompressionMethod, Error);

48
src/libutil/config.cc
View file

@ -1,6 +1,7 @@
#include "config.hh"
#include "args.hh"
#include "abstract-setting-to-json.hh"
#include "experimental-features.hh"
#include <nlohmann/json.hpp>
@ -80,16 +81,16 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
unsigned int pos = 0;
while (pos < contents.size()) {
string line;
std::string line;
while (pos < contents.size() && contents[pos] != '\n')
line += contents[pos++];
pos++;
string::size_type hash = line.find('#');
if (hash != string::npos)
line = string(line, 0, hash);
auto hash = line.find('#');
if (hash != std::string::npos)
line = std::string(line, 0, hash);
vector<string> tokens = tokenizeString<vector<string> >(line);
auto tokens = tokenizeString<std::vector<std::string>>(line);
if (tokens.empty()) continue;
if (tokens.size() < 2)
@ -119,9 +120,9 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
if (tokens[1] != "=")
throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
string name = tokens[0];
std::string name = tokens[0];
vector<string>::iterator i = tokens.begin();
auto i = tokens.begin();
advance(i, 2);
set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow
@ -131,7 +132,7 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
void AbstractConfig::applyConfigFile(const Path & path)
{
try {
string contents = readFile(path);
std::string contents = readFile(path);
applyConfig(contents, path);
} catch (SysError &) { }
}
@ -177,11 +178,6 @@ AbstractSetting::AbstractSetting(
{
}
void AbstractSetting::setDefault(const std::string & str)
{
if (!overridden) set(str);
}
nlohmann::json AbstractSetting::toJSON()
{
return nlohmann::json(toJSONObject());
@ -318,6 +314,31 @@ template<> std::string BaseSetting<StringSet>::to_string() const
return concatStringsSep(" ", value);
}
template<> void BaseSetting<std::set<ExperimentalFeature>>::set(const std::string & str, bool append)
{
if (!append) value.clear();
for (auto & s : tokenizeString<StringSet>(str)) {
auto thisXpFeature = parseExperimentalFeature(s);
if (thisXpFeature)
value.insert(thisXpFeature.value());
else
warn("unknown experimental feature '%s'", s);
}
}
template<> bool BaseSetting<std::set<ExperimentalFeature>>::isAppendable()
{
return true;
}
template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() const
{
StringSet stringifiedXpFeatures;
for (auto & feature : value)
stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature)));
return concatStringsSep(" ", stringifiedXpFeatures);
}
template<> void BaseSetting<StringMap>::set(const std::string & str, bool append)
{
if (!append) value.clear();
@ -353,6 +374,7 @@ template class BaseSetting<std::string>;
template class BaseSetting<Strings>;
template class BaseSetting<StringSet>;
template class BaseSetting<StringMap>;
template class BaseSetting<std::set<ExperimentalFeature>>;
void PathSetting::set(const std::string & str, bool append)
{

13
src/libutil/config.hh
View file

@ -194,8 +194,6 @@ public:
bool overridden = false;
void setDefault(const std::string & str);
protected:
AbstractSetting(
@ -234,16 +232,19 @@ protected:
T value;
const T defaultValue;
const bool documentDefault;
public:
BaseSetting(const T & def,
const bool documentDefault,
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases = {})
: AbstractSetting(name, description, aliases)
, value(def)
, defaultValue(def)
, documentDefault(documentDefault)
{ }
operator const T &() const { return value; }
@ -253,6 +254,7 @@ public:
bool operator !=(const T & v2) const { return value != v2; }
void operator =(const T & v) { assign(v); }
virtual void assign(const T & v) { value = v; }
void setDefault(const T & v) { if (!overridden) value = v; }
void set(const std::string & str, bool append = false) override;
@ -289,8 +291,9 @@ public:
const T & def,
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases = {})
: BaseSetting<T>(def, name, description, aliases)
const std::set<std::string> & aliases = {},
const bool documentDefault = true)
: BaseSetting<T>(def, documentDefault, name, description, aliases)
{
options->addSetting(this);
}
@ -312,7 +315,7 @@ public:
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases = {})
: BaseSetting<Path>(def, name, description, aliases)
: BaseSetting<Path>(def, true, name, description, aliases)
, allowEmpty(allowEmpty)
{
options->addSetting(this);

14
src/libutil/error.cc
View file

@ -17,7 +17,7 @@ BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
// c++ std::exception descendants must have a 'const char* what()' function.
// This stringifies the error and caches it for use by what(), or similarly by msg().
const string & BaseError::calcWhat() const
const std::string & BaseError::calcWhat() const
{
if (what_.has_value())
return *what_;
@ -25,21 +25,21 @@ const string & BaseError::calcWhat() const
err.name = sname();
std::ostringstream oss;
showErrorInfo(oss, err, false);
showErrorInfo(oss, err, loggerSettings.showTrace);
what_ = oss.str();
return *what_;
}
}
std::optional<string> ErrorInfo::programName = std::nullopt;
std::optional<std::string> ErrorInfo::programName = std::nullopt;
std::ostream & operator<<(std::ostream & os, const hintformat & hf)
{
return os << hf.str();
}
string showErrPos(const ErrPos & errPos)
std::string showErrPos(const ErrPos & errPos)
{
if (errPos.line > 0) {
if (errPos.column > 0) {
@ -68,7 +68,7 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
// count the newlines.
int count = 0;
string line;
std::string line;
int pl = errPos.line - 1;
do
{
@ -100,7 +100,7 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
std::istringstream iss(errPos.file);
// count the newlines.
int count = 0;
string line;
std::string line;
int pl = errPos.line - 1;
LinesOfCode loc;
@ -132,7 +132,7 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
// print lines of code to the ostream, indicating the error column.
void printCodeLines(std::ostream & out,
const string & prefix,
const std::string & prefix,
const ErrPos & errPos,
const LinesOfCode & loc)
{

28
src/libutil/error.hh
View file

@ -61,16 +61,16 @@ typedef enum {
// the lines of code surrounding an error.
struct LinesOfCode {
std::optional<string> prevLineOfCode;
std::optional<string> errLineOfCode;
std::optional<string> nextLineOfCode;
std::optional<std::string> prevLineOfCode;
std::optional<std::string> errLineOfCode;
std::optional<std::string> nextLineOfCode;
};
// ErrPos indicates the location of an error in a nix file.
struct ErrPos {
int line = 0;
int column = 0;
string file;
std::string file;
FileOrigin origin;
operator bool() const
@ -80,7 +80,7 @@ struct ErrPos {
// convert from the Pos struct, found in libexpr.
template <class P>
ErrPos& operator=(const P &pos)
ErrPos & operator=(const P & pos)
{
origin = pos.origin;
line = pos.line;
@ -94,7 +94,7 @@ struct ErrPos {
}
template <class P>
ErrPos(const P &p)
ErrPos(const P & p)
{
*this = p;
}
@ -107,15 +107,15 @@ struct Trace {
struct ErrorInfo {
Verbosity level;
string name; // FIXME: rename
std::string name; // FIXME: rename
hintformat msg;
std::optional<ErrPos> errPos;
std::list<Trace> traces;
static std::optional<string> programName;
static std::optional<std::string> programName;
};
std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace);
std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace);
/* BaseError should generally not be caught, as it has Interrupted as
a subclass. Catch Error instead. */
@ -124,8 +124,8 @@ class BaseError : public std::exception
protected:
mutable ErrorInfo err;
mutable std::optional<string> what_;
const string& calcWhat() const;
mutable std::optional<std::string> what_;
const std::string & calcWhat() const;
public:
unsigned int status = 1; // exit status
@ -137,7 +137,7 @@ public:
{ }
template<typename... Args>
BaseError(const std::string & fs, const Args & ... args)
explicit BaseError(const std::string & fs, const Args & ... args)
: err { .level = lvlError, .msg = hintfmt(fs, args...) }
{ }
@ -162,11 +162,11 @@ public:
const char * what() const noexcept override { return calcWhat().c_str(); }
#endif
const string & msg() const { return calcWhat(); }
const std::string & msg() const { return calcWhat(); }
const ErrorInfo & info() const { calcWhat(); return err; }
template<typename... Args>
BaseError & addTrace(std::optional<ErrPos> e, const string &fs, const Args & ... args)
BaseError & addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args)
{
return addTrace(e, hintfmt(fs, args...));
}

61
src/libutil/experimental-features.cc Normal file
View file

@ -0,0 +1,61 @@
#include "experimental-features.hh"
#include "util.hh"
#include "nlohmann/json.hpp"
namespace nix {
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::CaDerivations, "ca-derivations" },
{ Xp::Flakes, "flakes" },
{ Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" },
{ Xp::NoUrlLiterals, "no-url-literals" },
{ Xp::AutoAllocateUids, "auto-allocate-uids" },
{ Xp::SystemdCgroup, "systemd-cgroup" },
};
const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
{
using ReverseXpMap = std::map<std::string_view, ExperimentalFeature>;
static auto reverseXpMap = []()
{
auto reverseXpMap = std::make_unique<ReverseXpMap>();
for (auto & [feature, name] : stringifiedXpFeatures)
(*reverseXpMap)[name] = feature;
return reverseXpMap;
}();
if (auto feature = get(*reverseXpMap, name))
return *feature;
else
return std::nullopt;
}
std::string_view showExperimentalFeature(const ExperimentalFeature feature)
{
return stringifiedXpFeatures.at(feature);
}
std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
{
std::set<ExperimentalFeature> res;
for (auto & rawFeature : rawFeatures) {
if (auto feature = parseExperimentalFeature(rawFeature))
res.insert(*feature);
}
return res;
}
MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature)
: Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature))
, missingFeature(feature)
{}
std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & feature)
{
return str << showExperimentalFeature(feature);
}
}

58
src/libutil/experimental-features.hh Normal file
View file

@ -0,0 +1,58 @@
#pragma once
#include "comparator.hh"
#include "error.hh"
#include "nlohmann/json_fwd.hpp"
#include "types.hh"
namespace nix {
/**
* The list of available experimental features.
*
* If you update this, don’t forget to also change the map defining their
* string representation in the corresponding `.cc` file.
**/
enum struct ExperimentalFeature
{
CaDerivations,
Flakes,
NixCommand,
RecursiveNix,
NoUrlLiterals,
AutoAllocateUids,
SystemdCgroup,
};
/**
* Just because writing `ExperimentalFeature::CaDerivations` is way too long
*/
using Xp = ExperimentalFeature;
const std::optional<ExperimentalFeature> parseExperimentalFeature(
const std::string_view & name);
std::string_view showExperimentalFeature(const ExperimentalFeature);
std::ostream & operator<<(
std::ostream & str,
const ExperimentalFeature & feature);
/**
* Parse a set of strings to the corresponding set of experimental features,
* ignoring (but warning for) any unkwown feature.
*/
std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> &);
class MissingExperimentalFeature : public Error
{
public:
ExperimentalFeature missingFeature;
MissingExperimentalFeature(ExperimentalFeature);
virtual const char * sname() const override
{
return "MissingExperimentalFeature";
}
};
}

46
src/libutil/fmt.cc Normal file
View file

@ -0,0 +1,46 @@
#include "fmt.hh"
#include <regex>
namespace nix {
std::string hiliteMatches(
std::string_view s,
std::vector<std::smatch> matches,
std::string_view prefix,
std::string_view postfix)
{
// Avoid copy on zero matches
if (matches.size() == 0)
return (std::string) s;
std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) {
return a.position() < b.position();
});
std::string out;
ssize_t last_end = 0;
for (auto it = matches.begin(); it != matches.end();) {
auto m = *it;
size_t start = m.position();
out.append(s.substr(last_end, m.position() - last_end));
// Merge continous matches
ssize_t end = start + m.length();
while (++it != matches.end() && (*it).position() <= end) {
auto n = *it;
ssize_t nend = start + (n.position() - start + n.length());
if (nend > end)
end = nend;
}
out.append(prefix);
out.append(s.substr(start, end - start));
out.append(postfix);
last_end = end;
}
out.append(s.substr(last_end));
return out;
}
}

20
src/libutil/fmt.hh
View file

@ -2,6 +2,7 @@
#include <boost/format.hpp>
#include <string>
#include <regex>
#include "ansicolor.hh"
@ -9,7 +10,6 @@ namespace nix {
/* Inherit some names from other namespaces for convenience. */
using std::string;
using boost::format;
@ -20,8 +20,8 @@ struct nop { template<typename... T> nop(T...) {} };
struct FormatOrString
{
string s;
FormatOrString(const string & s) : s(s) { };
std::string s;
FormatOrString(std::string s) : s(std::move(s)) { };
template<class F>
FormatOrString(const F & f) : s(f.str()) { };
FormatOrString(const char * s) : s(s) { };
@ -101,7 +101,7 @@ std::ostream & operator<<(std::ostream & out, const normaltxt<T> & y)
class hintformat
{
public:
hintformat(const string & format) : fmt(format)
hintformat(const std::string & format) : fmt(format)
{
fmt.exceptions(boost::io::all_error_bits ^
boost::io::too_many_args_bit ^
@ -154,4 +154,16 @@ inline hintformat hintfmt(std::string plain_string)
// we won't be receiving any args in this case, so just print the original string
return hintfmt("%s", normaltxt(plain_string));
}
/* Highlight all the given matches in the given string `s` by wrapping
them between `prefix` and `postfix`.
If some matches overlap, then their union will be wrapped rather
than the individual matches. */
std::string hiliteMatches(
std::string_view s,
std::vector<std::smatch> matches,
std::string_view prefix,
std::string_view postfix);
}

21
src/libutil/hash.cc
View file

@ -66,31 +66,31 @@ bool Hash::operator < (const Hash & h) const
}
const string base16Chars = "0123456789abcdef";
const std::string base16Chars = "0123456789abcdef";
static string printHash16(const Hash & hash)
static std::string printHash16(const Hash & hash)
{
char buf[hash.hashSize * 2];
for (unsigned int i = 0; i < hash.hashSize; i++) {
buf[i * 2] = base16Chars[hash.hash[i] >> 4];
buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f];
}
return string(buf, hash.hashSize * 2);
return std::string(buf, hash.hashSize * 2);
}
// omitted: E O U T
const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
const std::string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
static string printHash32(const Hash & hash)
static std::string printHash32(const Hash & hash)
{
assert(hash.hashSize);
size_t len = hash.base32Len();
assert(len);
string s;
std::string s;
s.reserve(len);
for (int n = (int) len - 1; n >= 0; n--) {
@ -107,7 +107,7 @@ static string printHash32(const Hash & hash)
}
string printHash16or32(const Hash & hash)
std::string printHash16or32(const Hash & hash)
{
assert(hash.type);
return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false);
@ -151,7 +151,8 @@ Hash Hash::parseSRI(std::string_view original) {
}
// Mutates the string to eliminate the prefixes when found
static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_view & rest) {
static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_view & rest)
{
bool isSRI = false;
// Parse the has type before the separater, if there was one.
@ -259,7 +260,7 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI)
throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type));
}
Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht)
Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht)
{
if (hashStr.empty()) {
if (!ht)
@ -402,7 +403,7 @@ HashType parseHashType(std::string_view s)
throw UsageError("unknown hash algorithm '%1%'", s);
}
string printHashType(HashType ht)
std::string printHashType(HashType ht)
{
switch (ht) {
case htMD5: return "md5";

8
src/libutil/hash.hh
View file

@ -20,7 +20,7 @@ const int sha512HashSize = 64;
extern std::set<std::string> hashTypes;
extern const string base32Chars;
extern const std::string base32Chars;
enum Base : int { Base64, Base32, Base16, SRI };
@ -107,10 +107,10 @@ public:
};
/* Helper that defaults empty hashes to the 0 hash. */
Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht);
Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht);
/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */
string printHash16or32(const Hash & hash);
std::string printHash16or32(const Hash & hash);
/* Compute the hash of the given string. */
Hash hashString(HashType ht, std::string_view s);
@ -135,7 +135,7 @@ HashType parseHashType(std::string_view s);
std::optional<HashType> parseHashTypeOpt(std::string_view s);
/* And the reverse. */
string printHashType(HashType ht);
std::string printHashType(HashType ht);
union Ctx;

42
src/libutil/json.cc
View file

@ -7,16 +7,38 @@ namespace nix {
void toJSON(std::ostream & str, const char * start, const char * end)
{
str << '"';
for (auto i = start; i != end; i++)
if (*i == '\"' || *i == '\\') str << '\\' << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else if (*i >= 0 && *i < 32)
str << "\\u" << std::setfill('0') << std::setw(4) << std::hex << (uint16_t) *i << std::dec;
else str << *i;
str << '"';
constexpr size_t BUF_SIZE = 4096;
char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts
size_t bufPos = 0;
const auto flush = [&] {
str.write(buf, bufPos);
bufPos = 0;
};
const auto put = [&] (char c) {
buf[bufPos++] = c;
};
put('"');
for (auto i = start; i != end; i++) {
if (bufPos >= BUF_SIZE) flush();
if (*i == '\"' || *i == '\\') { put('\\'); put(*i); }
else if (*i == '\n') { put('\\'); put('n'); }
else if (*i == '\r') { put('\\'); put('r'); }
else if (*i == '\t') { put('\\'); put('t'); }
else if (*i >= 0 && *i < 32) {
const char hex[17] = "0123456789abcdef";
put('\\');
put('u');
put(hex[(uint16_t(*i) >> 12) & 0xf]);
put(hex[(uint16_t(*i) >> 8) & 0xf]);
put(hex[(uint16_t(*i) >> 4) & 0xf]);
put(hex[(uint16_t(*i) >> 0) & 0xf]);
}
else put(*i);
}
put('"');
flush();
}
void toJSON(std::ostream & str, const char * s)

2
src/libutil/logging.cc
View file

@ -113,7 +113,7 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs)
}
}
void writeToStderr(const string & s)
void writeToStderr(std::string_view s)
{
try {
writeFull(STDERR_FILENO, s, false);

9
src/libutil/logging.hh
View file

@ -40,7 +40,7 @@ struct LoggerSettings : Config
Setting<bool> showTrace{
this, false, "show-trace",
R"(
Where Nix should print out a stack trace in case of Nix
Whether Nix should print out a stack trace in case of Nix
expression evaluation errors.
)"};
};
@ -189,13 +189,14 @@ extern Verbosity verbosity; /* suppress msgs > this */
/* Print a string message if the current log level is at least the specified
level. Note that this has to be implemented as a macro to ensure that the
arguments are evaluated lazily. */
#define printMsg(level, args...) \
#define printMsgUsing(loggerParam, level, args...) \
do { \
auto __lvl = level; \
if (__lvl <= nix::verbosity) { \
logger->log(__lvl, fmt(args)); \
loggerParam->log(__lvl, fmt(args)); \
} \
} while (0)
#define printMsg(level, args...) printMsgUsing(logger, level, args)
#define printError(args...) printMsg(lvlError, args)
#define notice(args...) printMsg(lvlNotice, args)
@ -215,6 +216,6 @@ inline void warn(const std::string & fs, const Args & ... args)
void warnOnce(bool & haveWarned, const FormatOrString & fs);
void writeToStderr(const string & s);
void writeToStderr(std::string_view s);
}

2
src/libutil/ref.hh
View file

@ -17,7 +17,7 @@ private:
public:
ref<T>(const ref<T> & r)
ref(const ref<T> & r)
: p(r.p)
{ }

24
src/libutil/rust-ffi.cc
View file

@ -1,24 +0,0 @@
#if 0
#include "logging.hh"
#include "rust-ffi.hh"
extern "C" std::exception_ptr * make_error(rust::StringSlice s)
{
return new std::exception_ptr(std::make_exception_ptr(nix::Error(std::string(s.ptr, s.size))));
}
extern "C" void destroy_error(std::exception_ptr * ex)
{
free(ex);
}
namespace rust {
std::ostream & operator << (std::ostream & str, const String & s)
{
str << (std::string_view) s;
return str;
}
}
#endif

189
src/libutil/rust-ffi.hh
View file

@ -1,189 +0,0 @@
#pragma once
#if 0
#include "serialise.hh"
#include <string_view>
#include <cstring>
#include <array>
namespace rust {
typedef void (*DropFun)(void *);
/* A Rust value of N bytes. It can be moved but not copied. When it
goes out of scope, the C++ destructor will run the drop
function. */
template<std::size_t N, DropFun drop>
struct Value
{
protected:
std::array<char, N> raw;
~Value()
{
if (!isEvacuated()) {
drop(this);
evacuate();
}
}
// Must not be called directly.
Value()
{ }
Value(Value && other)
: raw(other.raw)
{
other.evacuate();
}
void operator =(Value && other)
{
if (!isEvacuated())
drop(this);
raw = other.raw;
other.evacuate();
}
private:
/* FIXME: optimize these (ideally in such a way that the compiler
can elide most calls to evacuate() / isEvacuated(). */
inline void evacuate()
{
for (auto & i : raw) i = 0;
}
inline bool isEvacuated()
{
for (auto & i : raw)
if (i != 0) return false;
return true;
}
};
/* A Rust vector. */
template<typename T, DropFun drop>
struct Vec : Value<3 * sizeof(void *), drop>
{
inline size_t size() const
{
return ((const size_t *) &this->raw)[2];
}
const T * data() const
{
return ((const T * *) &this->raw)[0];
}
};
/* A Rust slice. */
template<typename T>
struct Slice
{
const T * ptr;
size_t size;
Slice(const T * ptr, size_t size) : ptr(ptr), size(size)
{
assert(ptr);
}
};
struct StringSlice : Slice<char>
{
StringSlice(const std::string & s): Slice(s.data(), s.size()) {}
explicit StringSlice(std::string_view s): Slice(s.data(), s.size()) {}
StringSlice(const char * s): Slice(s, strlen(s)) {}
operator std::string_view() const
{
return std::string_view(ptr, size);
}
};
/* A Rust string. */
struct String;
extern "C" {
void ffi_String_new(StringSlice s, String * out);
void ffi_String_drop(void * s);
}
struct String : Vec<char, ffi_String_drop>
{
String() = delete;
String(std::string_view s)
{
ffi_String_new(StringSlice(s), this);
}
String(const char * s)
: String({s, std::strlen(s)})
{
}
operator std::string_view() const
{
return std::string_view(data(), size());
}
};
std::ostream & operator << (std::ostream & str, const String & s);
/* C++ representation of Rust's Result<T, CppException>. */
template<typename T>
struct Result
{
enum { Ok = 0, Err = 1, Uninit = 2 } tag;
union {
T data;
std::exception_ptr * exc;
};
Result() : tag(Uninit) { }; // FIXME: remove
Result(const Result &) = delete;
Result(Result && other)
: tag(other.tag)
{
other.tag = Uninit;
if (tag == Ok)
data = std::move(other.data);
else if (tag == Err)
exc = other.exc;
}
~Result()
{
if (tag == Ok)
data.~T();
else if (tag == Err)
free(exc);
else if (tag == Uninit)
;
else
abort();
}
/* Rethrow the wrapped exception or return the wrapped value. */
T unwrap()
{
if (tag == Ok) {
tag = Uninit;
return std::move(data);
}
else if (tag == Err)
std::rethrow_exception(*exc);
else
abort();
}
};
}
#endif

12
src/libutil/serialise.cc
View file

@ -110,7 +110,7 @@ std::string Source::drain()
{
StringSink s;
drainInto(s);
return *s.s;
return std::move(s.s);
}
@ -325,7 +325,7 @@ void writeString(std::string_view data, Sink & sink)
}
Sink & operator << (Sink & sink, const string & s)
Sink & operator << (Sink & sink, std::string_view s)
{
writeString(s, sink);
return sink;
@ -391,7 +391,7 @@ size_t readString(char * buf, size_t max, Source & source)
}
string readString(Source & source, size_t max)
std::string readString(Source & source, size_t max)
{
auto len = readNum<size_t>(source);
if (len > max) throw SerialisationError("string is too long");
@ -401,7 +401,7 @@ string readString(Source & source, size_t max)
return res;
}
Source & operator >> (Source & in, string & s)
Source & operator >> (Source & in, std::string & s)
{
s = readString(in);
return in;
@ -450,11 +450,11 @@ Error readError(Source & source)
void StringSink::operator () (std::string_view data)
{
static bool warned = false;
if (!warned && s->size() > threshold) {
if (!warned && s.size() > threshold) {
warnLargeDump();
warned = true;
}
s->append(data);
s.append(data);
}
size_t ChainSource::read(char * data, size_t len)

23
src/libutil/serialise.hh
View file

@ -154,12 +154,13 @@ private:
/* A sink that writes data to a string. */
struct StringSink : Sink
{
ref<std::string> s;
StringSink() : s(make_ref<std::string>()) { };
explicit StringSink(const size_t reservedSize) : s(make_ref<std::string>()) {
s->reserve(reservedSize);
std::string s;
StringSink() { }
explicit StringSink(const size_t reservedSize)
{
s.reserve(reservedSize);
};
StringSink(ref<std::string> s) : s(s) { };
StringSink(std::string && s) : s(std::move(s)) { };
void operator () (std::string_view data) override;
};
@ -167,9 +168,9 @@ struct StringSink : Sink
/* A source that reads data from a string. */
struct StringSource : Source
{
const string & s;
std::string_view s;
size_t pos;
StringSource(const string & _s) : s(_s), pos(0) { }
StringSource(std::string_view s) : s(s), pos(0) { }
size_t read(char * data, size_t len) override;
};
@ -317,10 +318,10 @@ inline Sink & operator << (Sink & sink, uint64_t n)
return sink;
}
Sink & operator << (Sink & sink, const string & s);
Sink & operator << (Sink & in, const Error & ex);
Sink & operator << (Sink & sink, std::string_view s);
Sink & operator << (Sink & sink, const Strings & s);
Sink & operator << (Sink & sink, const StringSet & s);
Sink & operator << (Sink & in, const Error & ex);
MakeError(SerialisationError, Error);
@ -363,10 +364,10 @@ inline uint64_t readLongLong(Source & source)
void readPadding(size_t len, Source & source);
size_t readString(char * buf, size_t max, Source & source);
string readString(Source & source, size_t max = std::numeric_limits<size_t>::max());
std::string readString(Source & source, size_t max = std::numeric_limits<size_t>::max());
template<class T> T readStrings(Source & source);
Source & operator >> (Source & in, string & s);
Source & operator >> (Source & in, std::string & s);
template<typename T>
Source & operator >> (Source & in, T & n)

9
src/libutil/tarfile.cc
View file

@ -93,9 +93,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
else
archive.check(r);
archive_entry_set_pathname(entry,
archive_entry_copy_pathname(entry,
(destDir + "/" + name).c_str());
// Patch hardlink path
const char *original_hardlink = archive_entry_hardlink(entry);
if (original_hardlink) {
archive_entry_copy_hardlink(entry,
(destDir + "/" + original_hardlink).c_str());
}
archive.check(archive_read_extract(archive.archive, entry, flags));
}

28
src/libutil/tests/compression.cc
View file

@ -12,17 +12,17 @@ namespace nix {
}
TEST(compress, noneMethodDoesNothingToTheInput) {
ref<std::string> o = compress("none", "this-is-a-test");
auto o = compress("none", "this-is-a-test");
ASSERT_EQ(*o, "this-is-a-test");
ASSERT_EQ(o, "this-is-a-test");
}
TEST(decompress, decompressNoneCompressed) {
auto method = "none";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, str);
auto o = decompress(method, str);
ASSERT_EQ(*o, str);
ASSERT_EQ(o, str);
}
TEST(decompress, decompressEmptyCompressed) {
@ -30,33 +30,33 @@ namespace nix {
// (Content-Encoding == "").
auto method = "";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, str);
auto o = decompress(method, str);
ASSERT_EQ(*o, str);
ASSERT_EQ(o, str);
}
TEST(decompress, decompressXzCompressed) {
auto method = "xz";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, *compress(method, str));
auto o = decompress(method, compress(method, str));
ASSERT_EQ(*o, str);
ASSERT_EQ(o, str);
}
TEST(decompress, decompressBzip2Compressed) {
auto method = "bzip2";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, *compress(method, str));
auto o = decompress(method, compress(method, str));
ASSERT_EQ(*o, str);
ASSERT_EQ(o, str);
}
TEST(decompress, decompressBrCompressed) {
auto method = "br";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, *compress(method, str));
auto o = decompress(method, compress(method, str));
ASSERT_EQ(*o, str);
ASSERT_EQ(o, str);
}
TEST(decompress, decompressInvalidInputThrowsCompressionError) {
@ -77,7 +77,7 @@ namespace nix {
(*sink)(inputString);
sink->finish();
ASSERT_STREQ((*strSink.s).c_str(), inputString);
ASSERT_STREQ(strSink.s.c_str(), inputString);
}
TEST(makeCompressionSink, compressAndDecompress) {
@ -90,7 +90,7 @@ namespace nix {
sink->finish();
decompressionSink->finish();
ASSERT_STREQ((*strSink.s).c_str(), inputString);
ASSERT_STREQ(strSink.s.c_str(), inputString);
}
}

2
src/libutil/tests/config.cc
View file

@ -161,7 +161,7 @@ namespace nix {
Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
setting.assign("value");
ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#");
ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#");
}
TEST(Config, setSettingAlias) {

68
src/libutil/tests/fmt.cc Normal file
View file

@ -0,0 +1,68 @@
#include "fmt.hh"
#include <gtest/gtest.h>
#include <regex>
namespace nix {
/* ----------- tests for fmt.hh -------------------------------------------------*/
TEST(hiliteMatches, noHighlight) {
ASSERT_STREQ(hiliteMatches("Hello, world!", std::vector<std::smatch>(), "(", ")").c_str(), "Hello, world!");
}
TEST(hiliteMatches, simpleHighlight) {
std::string str = "Hello, world!";
std::regex re = std::regex("world");
auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator());
ASSERT_STREQ(
hiliteMatches(str, matches, "(", ")").c_str(),
"Hello, (world)!"
);
}
TEST(hiliteMatches, multipleMatches) {
std::string str = "Hello, world, world, world, world, world, world, Hello!";
std::regex re = std::regex("world");
auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator());
ASSERT_STREQ(
hiliteMatches(str, matches, "(", ")").c_str(),
"Hello, (world), (world), (world), (world), (world), (world), Hello!"
);
}
TEST(hiliteMatches, overlappingMatches) {
std::string str = "world, Hello, world, Hello, world, Hello, world, Hello, world!";
std::regex re = std::regex("Hello, world");
std::regex re2 = std::regex("world, Hello");
auto v = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator());
for(auto it = std::sregex_iterator(str.begin(), str.end(), re2); it != std::sregex_iterator(); ++it) {
v.push_back(*it);
}
ASSERT_STREQ(
hiliteMatches(str, v, "(", ")").c_str(),
"(world, Hello, world, Hello, world, Hello, world, Hello, world)!"
);
}
TEST(hiliteMatches, complexOverlappingMatches) {
std::string str = "legacyPackages.x86_64-linux.git-crypt";
std::vector regexes = {
std::regex("t-cry"),
std::regex("ux\\.git-cry"),
std::regex("git-c"),
std::regex("pt"),
};
std::vector<std::smatch> matches;
for(auto regex : regexes)
{
for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) {
matches.push_back(*it);
}
}
ASSERT_STREQ(
hiliteMatches(str, matches, "(", ")").c_str(),
"legacyPackages.x86_64-lin(ux.git-crypt)"
);
}
}

17
src/libutil/tests/tests.cc
View file

@ -4,6 +4,8 @@
#include <limits.h>
#include <gtest/gtest.h>
#include <numeric>
namespace nix {
/* ----------- tests for util.hh ------------------------------------------------*/
@ -282,6 +284,17 @@ namespace nix {
ASSERT_EQ(decoded, s);
}
TEST(base64Encode, encodeAndDecodeNonPrintable) {
char s[256];
std::iota(std::rbegin(s), std::rend(s), 0);
auto encoded = base64Encode(s);
auto decoded = base64Decode(encoded);
EXPECT_EQ(decoded.length(), 255);
ASSERT_EQ(decoded, s);
}
/* ----------------------------------------------------------------------------
* base64Decode
* --------------------------------------------------------------------------*/
@ -294,6 +307,10 @@ namespace nix {
ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum");
}
TEST(base64Decode, decodeThrowsOnInvalidChar) {
ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error);
}
/* ----------------------------------------------------------------------------
* toLower
* --------------------------------------------------------------------------*/

18
src/libutil/tests/url.cc
View file

@ -5,9 +5,9 @@ namespace nix {
/* ----------- tests for url.hh --------------------------------------------------*/
string print_map(std::map<string, string> m) {
std::map<string, string>::iterator it;
string s = "{ ";
std::string print_map(std::map<std::string, std::string> m) {
std::map<std::string, std::string>::iterator it;
std::string s = "{ ";
for (it = m.begin(); it != m.end(); ++it) {
s += "{ ";
s += it->first;
@ -262,21 +262,21 @@ namespace nix {
* --------------------------------------------------------------------------*/
TEST(percentDecode, decodesUrlEncodedString) {
string s = "==@==";
string d = percentDecode("%3D%3D%40%3D%3D");
std::string s = "==@==";
std::string d = percentDecode("%3D%3D%40%3D%3D");
ASSERT_EQ(d, s);
}
TEST(percentDecode, multipleDecodesAreIdempotent) {
string once = percentDecode("%3D%3D%40%3D%3D");
string twice = percentDecode(once);
std::string once = percentDecode("%3D%3D%40%3D%3D");
std::string twice = percentDecode(once);
ASSERT_EQ(once, twice);
}
TEST(percentDecode, trailingPercent) {
string s = "==@==%";
string d = percentDecode("%3D%3D%40%3D%3D%25");
std::string s = "==@==%";
std::string d = percentDecode("%3D%3D%40%3D%3D%25");
ASSERT_EQ(d, s);
}

3
src/libutil/thread-pool.cc
View file

@ -1,13 +1,10 @@
#include "thread-pool.hh"
#include "affinity.hh"
namespace nix {
ThreadPool::ThreadPool(size_t _maxThreads)
: maxThreads(_maxThreads)
{
restoreAffinity(); // FIXME
if (!maxThreads) {
maxThreads = std::thread::hardware_concurrency();
if (!maxThreads) maxThreads = 1;

82
src/libutil/types.hh
View file

@ -6,26 +6,23 @@
#include <set>
#include <string>
#include <map>
#include <variant>
#include <vector>
namespace nix {
using std::list;
using std::set;
using std::vector;
using std::string;
typedef list<string> Strings;
typedef set<string> StringSet;
typedef std::map<string, string> StringMap;
typedef std::list<std::string> Strings;
typedef std::set<std::string> StringSet;
typedef std::map<std::string, std::string> StringMap;
typedef std::map<std::string, std::string> StringPairs;
/* Paths are just strings. */
typedef std::string Path;
typedef std::string_view PathView;
typedef std::list<Path> Paths;
typedef std::set<Path> PathSet;
typedef string Path;
typedef list<Path> Paths;
typedef set<Path> PathSet;
typedef vector<std::pair<string, string>> Headers;
typedef std::vector<std::pair<std::string, std::string>> Headers;
/* Helper class to run code at startup. */
template<typename T>
@ -46,4 +43,63 @@ struct Explicit {
}
};
/* This wants to be a little bit like rust's Cow type.
Some parts of the evaluator benefit greatly from being able to reuse
existing allocations for strings, but have to be able to also use
newly allocated storage for values.
We do not define implicit conversions, even with ref qualifiers,
since those can easily become ambiguous to the reader and can degrade
into copying behaviour we want to avoid. */
class BackedStringView {
private:
std::variant<std::string, std::string_view> data;
/* Needed to introduce a temporary since operator-> must return
a pointer. Without this we'd need to store the view object
even when we already own a string. */
class Ptr {
private:
std::string_view view;
public:
Ptr(std::string_view view): view(view) {}
const std::string_view * operator->() const { return &view; }
};
public:
BackedStringView(std::string && s): data(std::move(s)) {}
BackedStringView(std::string_view sv): data(sv) {}
template<size_t N>
BackedStringView(const char (& lit)[N]): data(std::string_view(lit)) {}
BackedStringView(const BackedStringView &) = delete;
BackedStringView & operator=(const BackedStringView &) = delete;
/* We only want move operations defined since the sole purpose of
this type is to avoid copies. */
BackedStringView(BackedStringView && other) = default;
BackedStringView & operator=(BackedStringView && other) = default;
bool isOwned() const
{
return std::holds_alternative<std::string>(data);
}
std::string toOwned() &&
{
return isOwned()
? std::move(std::get<std::string>(data))
: std::string(std::get<std::string_view>(data));
}
std::string_view operator*() const
{
return isOwned()
? std::get<std::string>(data)
: std::get<std::string_view>(data);
}
Ptr operator->() const { return Ptr(**this); }
};
}

366
src/libutil/util.cc
View file

@ -1,5 +1,4 @@
#include "util.hh"
#include "affinity.hh"
#include "sync.hh"
#include "finally.hh"
#include "serialise.hh"
@ -82,7 +81,7 @@ void replaceEnv(std::map<std::string, std::string> newEnv)
}
Path absPath(Path path, std::optional<Path> dir, bool resolveSymlinks)
Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
{
if (path[0] != '/') {
if (!dir) {
@ -96,28 +95,28 @@ Path absPath(Path path, std::optional<Path> dir, bool resolveSymlinks)
if (!getcwd(buf, sizeof(buf)))
#endif
throw SysError("cannot get cwd");
dir = buf;
path = concatStrings(buf, "/", path);
#ifdef __GNU__
free(buf);
#endif
}
path = *dir + "/" + path;
} else
path = concatStrings(*dir, "/", path);
}
return canonPath(path, resolveSymlinks);
}
Path canonPath(const Path & path, bool resolveSymlinks)
Path canonPath(PathView path, bool resolveSymlinks)
{
assert(path != "");
string s;
std::string s;
s.reserve(256);
if (path[0] != '/')
throw Error("not an absolute path: '%1%'", path);
string::const_iterator i = path.begin(), end = path.end();
string temp;
std::string temp;
/* Count the number of times we follow a symlink and stop at some
arbitrary (but high) limit to prevent infinite loops. */
@ -126,33 +125,37 @@ Path canonPath(const Path & path, bool resolveSymlinks)
while (1) {
/* Skip slashes. */
while (i != end && *i == '/') i++;
if (i == end) break;
while (!path.empty() && path[0] == '/') path.remove_prefix(1);
if (path.empty()) break;
/* Ignore `.'. */
if (*i == '.' && (i + 1 == end || i[1] == '/'))
i++;
if (path == "." || path.substr(0, 2) == "./")
path.remove_prefix(1);
/* If `..', delete the last component. */
else if (*i == '.' && i + 1 < end && i[1] == '.' &&
(i + 2 == end || i[2] == '/'))
else if (path == ".." || path.substr(0, 3) == "../")
{
if (!s.empty()) s.erase(s.rfind('/'));
i += 2;
path.remove_prefix(2);
}
/* Normal component; copy it. */
else {
s += '/';
while (i != end && *i != '/') s += *i++;
if (const auto slash = path.find('/'); slash == std::string::npos) {
s += path;
path = {};
} else {
s += path.substr(0, slash);
path = path.substr(slash);
}
/* If s points to a symlink, resolve it and continue from there */
if (resolveSymlinks && isLink(s)) {
if (++followCount >= maxFollow)
throw Error("infinite symlink recursion in path '%1%'", path);
temp = readLink(s) + string(i, end);
i = temp.begin();
end = temp.end();
temp = concatStrings(readLink(s), path);
path = temp;
if (!temp.empty() && temp[0] == '/') {
s.clear(); /* restart for symlinks pointing to absolute path */
} else {
@ -165,14 +168,14 @@ Path canonPath(const Path & path, bool resolveSymlinks)
}
}
return s.empty() ? "/" : s;
return s.empty() ? "/" : std::move(s);
}
Path dirOf(const Path & path)
Path dirOf(const PathView path)
{
Path::size_type pos = path.rfind('/');
if (pos == string::npos)
if (pos == std::string::npos)
return ".";
return pos == 0 ? "/" : Path(path, 0, pos);
}
@ -188,7 +191,7 @@ std::string_view baseNameOf(std::string_view path)
last -= 1;
auto pos = path.rfind('/', last);
if (pos == string::npos)
if (pos == std::string::npos)
pos = 0;
else
pos += 1;
@ -197,16 +200,16 @@ std::string_view baseNameOf(std::string_view path)
}
bool isInDir(const Path & path, const Path & dir)
bool isInDir(std::string_view path, std::string_view dir)
{
return path[0] == '/'
&& string(path, 0, dir.size()) == dir
return path.substr(0, 1) == "/"
&& path.substr(0, dir.size()) == dir
&& path.size() >= dir.size() + 2
&& path[dir.size()] == '/';
}
bool isDirOrInDir(const Path & path, const Path & dir)
bool isDirOrInDir(std::string_view path, std::string_view dir)
{
return path == dir || isInDir(path, dir);
}
@ -246,7 +249,7 @@ Path readLink(const Path & path)
else
throw SysError("reading symbolic link '%1%'", path);
else if (rlSize < bufSize)
return string(buf.data(), rlSize);
return std::string(buf.data(), rlSize);
}
}
@ -266,7 +269,7 @@ DirEntries readDirectory(DIR *dir, const Path & path)
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir)) { /* sic */
checkInterrupt();
string name = dirent->d_name;
std::string name = dirent->d_name;
if (name == "." || name == "..") continue;
entries.emplace_back(name, dirent->d_ino,
#ifdef HAVE_STRUCT_DIRENT_D_TYPE
@ -300,7 +303,7 @@ unsigned char getFileType(const Path & path)
}
string readFile(int fd)
std::string readFile(int fd)
{
struct stat st;
if (fstat(fd, &st) == -1)
@ -310,7 +313,7 @@ string readFile(int fd)
}
string readFile(const Path & path)
std::string readFile(const Path & path)
{
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd)
@ -363,9 +366,9 @@ void writeFile(const Path & path, Source & source, mode_t mode)
}
}
string readLine(int fd)
std::string readLine(int fd)
{
string s;
std::string s;
while (1) {
checkInterrupt();
char ch;
@ -384,7 +387,7 @@ string readLine(int fd)
}
void writeLine(int fd, string s)
void writeLine(int fd, std::string s)
{
s += '\n';
writeFull(fd, s);
@ -395,7 +398,7 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
{
checkInterrupt();
string name(baseNameOf(path));
std::string name(baseNameOf(path));
struct stat st;
if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
@ -512,6 +515,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
if (!fd)
throw SysError("creating temporary file '%s'", tmpl);
closeOnExec(fd.get());
return {std::move(fd), tmpl};
}
@ -562,8 +566,8 @@ Path getConfigDir()
std::vector<Path> getConfigDirs()
{
Path configHome = getConfigDir();
string configDirs = getEnv("XDG_CONFIG_DIRS").value_or("");
std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":");
auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg");
std::vector<Path> result = tokenizeString<std::vector<std::string>>(configDirs, ":");
result.insert(result.begin(), configHome);
return result;
}
@ -666,11 +670,13 @@ void writeFull(int fd, std::string_view s, bool allowInterrupts)
}
string drainFD(int fd, bool block, const size_t reserveSize)
std::string drainFD(int fd, bool block, const size_t reserveSize)
{
StringSink sink(reserveSize);
// the parser needs two extra bytes to append terminating characters, other users will
// not care very much about the extra memory.
StringSink sink(reserveSize + 2);
drainFD(fd, sink, block);
return std::move(*sink.s);
return std::move(sink.s);
}
@ -713,7 +719,7 @@ void drainFD(int fd, Sink & sink, bool block)
AutoDelete::AutoDelete() : del{false} {}
AutoDelete::AutoDelete(const string & p, bool recursive) : path(p)
AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p)
{
del = true;
this->recursive = recursive;
@ -903,7 +909,7 @@ int Pid::wait()
return status;
}
if (errno != EINTR)
throw SysError("cannot get child exit status");
throw SysError("cannot get exit status of PID %d", pid);
checkInterrupt();
}
}
@ -939,9 +945,6 @@ void killUser(uid_t uid)
users to which the current process can send signals. So we
fork a process, switch to uid, and send a mass kill. */
ProcessOptions options;
options.allowVfork = false;
Pid pid = startProcess([&]() {
if (setuid(uid) == -1)
@ -964,7 +967,7 @@ void killUser(uid_t uid)
}
_exit(0);
}, options);
});
int status = pid.wait();
if (status != 0)
@ -1006,7 +1009,6 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
throw SysError("setting death signal");
#endif
restoreAffinity();
fun();
} catch (std::exception & e) {
try {
@ -1034,7 +1036,7 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
return res;
}
string runProgram(Path program, bool searchPath, const Strings & args,
std::string runProgram(Path program, bool searchPath, const Strings & args,
const std::optional<std::string> & input)
{
auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
@ -1059,7 +1061,7 @@ std::pair<int, std::string> runProgram(RunOptions && options)
status = e.status;
}
return {status, std::move(*sink.s)};
return {status, std::move(sink.s)};
}
void runProgram2(const RunOptions & options)
@ -1085,8 +1087,7 @@ void runProgram2(const RunOptions & options)
// vfork implies that the environment of the main process and the fork will
// be shared (technically this is undefined, but in practice that's the
// case), so we can't use it if we alter the environment
if (options.environment)
processOptions.allowVfork = false;
processOptions.allowVfork = !options.environment;
/* Fork. */
Pid pid = startProcess([&]() {
@ -1173,7 +1174,7 @@ void runProgram2(const RunOptions & options)
}
void closeMostFDs(const set<int> & exceptions)
void closeMostFDs(const std::set<int> & exceptions)
{
#if __linux__
try {
@ -1209,7 +1210,7 @@ void closeOnExec(int fd)
//////////////////////////////////////////////////////////////////////
bool _isInterrupted = false;
std::atomic<bool> _isInterrupted = false;
static thread_local bool interruptThrown = false;
thread_local std::function<bool()> interruptCheck;
@ -1234,45 +1235,45 @@ void _interrupted()
//////////////////////////////////////////////////////////////////////
template<class C> C tokenizeString(std::string_view s, const string & separators)
template<class C> C tokenizeString(std::string_view s, std::string_view separators)
{
C result;
string::size_type pos = s.find_first_not_of(separators, 0);
while (pos != string::npos) {
string::size_type end = s.find_first_of(separators, pos + 1);
if (end == string::npos) end = s.size();
string token(s, pos, end - pos);
result.insert(result.end(), token);
auto pos = s.find_first_not_of(separators, 0);
while (pos != std::string::npos) {
auto end = s.find_first_of(separators, pos + 1);
if (end == std::string::npos) end = s.size();
result.insert(result.end(), std::string(s, pos, end - pos));
pos = s.find_first_not_of(separators, end);
}
return result;
}
template Strings tokenizeString(std::string_view s, const string & separators);
template StringSet tokenizeString(std::string_view s, const string & separators);
template vector<string> tokenizeString(std::string_view s, const string & separators);
template Strings tokenizeString(std::string_view s, std::string_view separators);
template StringSet tokenizeString(std::string_view s, std::string_view separators);
template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);
string chomp(std::string_view s)
std::string chomp(std::string_view s)
{
size_t i = s.find_last_not_of(" \n\r\t");
return i == string::npos ? "" : string(s, 0, i + 1);
return i == std::string_view::npos ? "" : std::string(s, 0, i + 1);
}
string trim(const string & s, const string & whitespace)
std::string trim(std::string_view s, std::string_view whitespace)
{
auto i = s.find_first_not_of(whitespace);
if (i == string::npos) return "";
if (i == std::string_view::npos) return "";
auto j = s.find_last_not_of(whitespace);
return string(s, i, j == string::npos ? j : j - i + 1);
return std::string(s, i, j == std::string::npos ? j : j - i + 1);
}
string replaceStrings(std::string_view s,
const std::string & from, const std::string & to)
std::string replaceStrings(
std::string res,
std::string_view from,
std::string_view to)
{
string res(s);
if (from.empty()) return res;
size_t pos = 0;
while ((pos = res.find(from, pos)) != std::string::npos) {
@ -1283,20 +1284,19 @@ string replaceStrings(std::string_view s,
}
std::string rewriteStrings(const std::string & _s, const StringMap & rewrites)
std::string rewriteStrings(std::string s, const StringMap & rewrites)
{
auto s = _s;
for (auto & i : rewrites) {
if (i.first == i.second) continue;
size_t j = 0;
while ((j = s.find(i.first, j)) != string::npos)
while ((j = s.find(i.first, j)) != std::string::npos)
s.replace(j, i.first.size(), i.second);
}
return s;
}
string statusToString(int status)
std::string statusToString(int status)
{
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
if (WIFEXITED(status))
@ -1344,9 +1344,11 @@ std::string toLower(const std::string & s)
}
std::string shellEscape(const std::string & s)
std::string shellEscape(const std::string_view s)
{
std::string r = "'";
std::string r;
r.reserve(s.size() + 2);
r += "'";
for (auto & i : s)
if (i == '\'') r += "'\\''"; else r += i;
r += '\'';
@ -1356,11 +1358,15 @@ std::string shellEscape(const std::string & s)
void ignoreException()
{
/* Make sure no exceptions leave this function.
printError() also throws when remote is closed. */
try {
throw;
} catch (std::exception & e) {
printError("error (ignored): %1%", e.what());
}
try {
throw;
} catch (std::exception & e) {
printError("error (ignored): %1%", e.what());
}
} catch (...) { }
}
bool shouldANSI()
@ -1440,12 +1446,11 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in
}
static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
static std::array<char, 256> base64DecodeChars;
constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
string base64Encode(std::string_view s)
std::string base64Encode(std::string_view s)
{
string res;
std::string res;
int data = 0, nbits = 0;
for (char c : s) {
@ -1464,16 +1469,19 @@ string base64Encode(std::string_view s)
}
string base64Decode(std::string_view s)
std::string base64Decode(std::string_view s)
{
static std::once_flag flag;
std::call_once(flag, [](){
base64DecodeChars = { (char)-1 };
constexpr char npos = -1;
constexpr std::array<char, 256> base64DecodeChars = [&]() {
std::array<char, 256> result{};
for (auto& c : result)
c = npos;
for (int i = 0; i < 64; i++)
base64DecodeChars[(int) base64Chars[i]] = i;
});
result[base64Chars[i]] = i;
return result;
}();
string res;
std::string res;
unsigned int d = 0, bits = 0;
for (char c : s) {
@ -1481,7 +1489,7 @@ string base64Decode(std::string_view s)
if (c == '\n') continue;
char digit = base64DecodeChars[(unsigned char) c];
if (digit == -1)
if (digit == npos)
throw Error("invalid character in Base64 string: '%c'", c);
bits += 6;
@ -1557,7 +1565,22 @@ std::pair<unsigned short, unsigned short> getWindowSize()
}
static Sync<std::list<std::function<void()>>> _interruptCallbacks;
/* We keep track of interrupt callbacks using integer tokens, so we can iterate
safely without having to lock the data structure while executing arbitrary
functions.
*/
struct InterruptCallbacks {
typedef int64_t Token;
/* We use unique tokens so that we can't accidentally delete the wrong
handler because of an erroneous double delete. */
Token nextToken = 0;
/* Used as a list, see InterruptCallbacks comment. */
std::map<Token, std::function<void()>> callbacks;
};
static Sync<InterruptCallbacks> _interruptCallbacks;
static void signalHandlerThread(sigset_t set)
{
@ -1579,8 +1602,19 @@ void triggerInterrupt()
_isInterrupted = true;
{
auto interruptCallbacks(_interruptCallbacks.lock());
for (auto & callback : *interruptCallbacks) {
InterruptCallbacks::Token i = 0;
while (true) {
std::function<void()> callback;
{
auto interruptCallbacks(_interruptCallbacks.lock());
auto lb = interruptCallbacks->callbacks.lower_bound(i);
if (lb == interruptCallbacks->callbacks.end())
break;
callback = lb->second;
i = lb->first + 1;
}
try {
callback();
} catch (...) {
@ -1634,11 +1668,47 @@ void setStackSize(size_t stackSize)
#endif
}
void restoreProcessContext()
static AutoCloseFD fdSavedMountNamespace;
void saveMountNamespace()
{
#if __linux__
static std::once_flag done;
std::call_once(done, []() {
AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY);
if (!fd)
throw SysError("saving parent mount namespace");
fdSavedMountNamespace = std::move(fd);
});
#endif
}
void restoreMountNamespace()
{
#if __linux__
try {
if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
throw SysError("restoring parent mount namespace");
} catch (Error & e) {
debug(e.msg());
}
#endif
}
void unshareFilesystem()
{
#ifdef __linux__
if (unshare(CLONE_FS) != 0 && errno != EPERM)
throw SysError("unsharing filesystem state in download thread");
#endif
}
void restoreProcessContext(bool restoreMounts)
{
restoreSignals();
restoreAffinity();
if (restoreMounts) {
restoreMountNamespace();
}
#if __linux__
if (savedStackSize) {
@ -1654,27 +1724,28 @@ void restoreProcessContext()
/* RAII helper to automatically deregister a callback. */
struct InterruptCallbackImpl : InterruptCallback
{
std::list<std::function<void()>>::iterator it;
InterruptCallbacks::Token token;
~InterruptCallbackImpl() override
{
_interruptCallbacks.lock()->erase(it);
auto interruptCallbacks(_interruptCallbacks.lock());
interruptCallbacks->callbacks.erase(token);
}
};
std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> callback)
{
auto interruptCallbacks(_interruptCallbacks.lock());
interruptCallbacks->push_back(callback);
auto token = interruptCallbacks->nextToken++;
interruptCallbacks->callbacks.emplace(token, callback);
auto res = std::make_unique<InterruptCallbackImpl>();
res->it = interruptCallbacks->end();
res->it--;
res->token = token;
return std::unique_ptr<InterruptCallback>(res.release());
}
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
AutoCloseFD createUnixDomainSocket()
{
AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM
#ifdef SOCK_CLOEXEC
@ -1683,19 +1754,16 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
, 0);
if (!fdSocket)
throw SysError("cannot create Unix domain socket");
closeOnExec(fdSocket.get());
return fdSocket;
}
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%1%' is too long", path);
strcpy(addr.sun_path, path.c_str());
unlink(path.c_str());
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
{
auto fdSocket = nix::createUnixDomainSocket();
if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%1%'", path);
bind(fdSocket.get(), path);
if (chmod(path.c_str(), mode) == -1)
throw SysError("changing permissions on '%1%'", path);
@ -1707,7 +1775,67 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
}
string showBytes(uint64_t bytes)
void bind(int fd, const std::string & path)
{
unlink(path.c_str());
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
Path dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot bind to socket '%s'", path);
} else {
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
}
}
void connect(int fd, const std::string & path)
{
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
Path dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot connect to socket at '%s'", path);
} else {
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
}
}
std::string showBytes(uint64_t bytes)
{
return fmt("%.2f MiB", bytes / (1024.0 * 1024.0));
}
@ -1716,8 +1844,10 @@ string showBytes(uint64_t bytes)
// FIXME: move to libstore/build
void commonChildInit(Pipe & logPipe)
{
const static string pathNullDevice = "/dev/null";
restoreProcessContext();
logger = makeSimpleLogger();
const static std::string pathNullDevice = "/dev/null";
restoreProcessContext(false);
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning

166
src/libutil/util.hh
View file

@ -11,6 +11,9 @@
#include <unistd.h>
#include <signal.h>
#include <boost/lexical_cast.hpp>
#include <atomic>
#include <functional>
#include <map>
#include <sstream>
@ -46,30 +49,32 @@ void clearEnv();
specified directory, or the current directory otherwise. The path
is also canonicalised. */
Path absPath(Path path,
std::optional<Path> dir = {},
std::optional<PathView> dir = {},
bool resolveSymlinks = false);
/* Canonicalise a path by removing all `.' or `..' components and
double or trailing slashes. Optionally resolves all symlink
components such that each component of the resulting path is *not*
a symbolic link. */
Path canonPath(const Path & path, bool resolveSymlinks = false);
Path canonPath(PathView path, bool resolveSymlinks = false);
/* Return the directory part of the given canonical path, i.e.,
everything before the final `/'. If the path is the root or an
immediate child thereof (e.g., `/foo'), this means `/'
is returned.*/
Path dirOf(const Path & path);
Path dirOf(const PathView path);
/* Return the base name of the given canonical path, i.e., everything
following the final `/' (trailing slashes are removed). */
std::string_view baseNameOf(std::string_view path);
/* Check whether 'path' is a descendant of 'dir'. */
bool isInDir(const Path & path, const Path & dir);
/* Check whether 'path' is a descendant of 'dir'. Both paths must be
canonicalized. */
bool isInDir(std::string_view path, std::string_view dir);
/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */
bool isDirOrInDir(const Path & path, const Path & dir);
/* Check whether 'path' is equal to 'dir' or a descendant of
'dir'. Both paths must be canonicalized. */
bool isDirOrInDir(std::string_view path, std::string_view dir);
/* Get status of `path'. */
struct stat lstat(const Path & path);
@ -87,22 +92,22 @@ bool isLink(const Path & path);
removed. */
struct DirEntry
{
string name;
std::string name;
ino_t ino;
unsigned char type; // one of DT_*
DirEntry(const string & name, ino_t ino, unsigned char type)
: name(name), ino(ino), type(type) { }
DirEntry(std::string name, ino_t ino, unsigned char type)
: name(std::move(name)), ino(ino), type(type) { }
};
typedef vector<DirEntry> DirEntries;
typedef std::vector<DirEntry> DirEntries;
DirEntries readDirectory(const Path & path);
unsigned char getFileType(const Path & path);
/* Read the contents of a file into a string. */
string readFile(int fd);
string readFile(const Path & path);
std::string readFile(int fd);
std::string readFile(const Path & path);
void readFile(const Path & path, Sink & sink);
/* Write a string to a file. */
@ -111,10 +116,10 @@ void writeFile(const Path & path, std::string_view s, mode_t mode = 0666);
void writeFile(const Path & path, Source & source, mode_t mode = 0666);
/* Read a line from a file descriptor. */
string readLine(int fd);
std::string readLine(int fd);
/* Write a line to a file descriptor. */
void writeLine(int fd, string s);
void writeLine(int fd, std::string s);
/* Delete a path; i.e., in the case of a directory, it is deleted
recursively. It's not an error if the path does not exist. The
@ -143,6 +148,9 @@ Path getDataDir();
/* Create a directory and all its parents, if necessary. Returns the
list of created directories, in order of creation. */
Paths createDirs(const Path & path);
inline Paths createDirs(PathView path) {
return createDirs(Path(path));
}
/* Create a symlink. */
void createSymlink(const Path & target, const Path & link,
@ -162,7 +170,7 @@ MakeError(EndOfFile, Error);
/* Read a file descriptor until EOF occurs. */
string drainFD(int fd, bool block = true, const size_t reserveSize=0);
std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
void drainFD(int fd, Sink & sink, bool block = true);
@ -182,6 +190,7 @@ public:
void cancel();
void reset(const Path & p, bool recursive = true);
operator Path() const { return path; }
operator PathView() const { return path; }
};
@ -259,10 +268,10 @@ void killUser(uid_t uid);
pid to the caller. */
struct ProcessOptions
{
string errorPrefix = "error: ";
std::string errorPrefix = "";
bool dieWithParent = true;
bool runExitHandlers = false;
bool allowVfork = true;
bool allowVfork = false;
};
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
@ -270,7 +279,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P
/* Run a program and return its stdout in a string (i.e., like the
shell backtick operator). */
string runProgram(Path program, bool searchPath = false,
std::string runProgram(Path program, bool searchPath = false,
const Strings & args = Strings(),
const std::optional<std::string> & input = {});
@ -299,8 +308,21 @@ void setStackSize(size_t stackSize);
/* Restore the original inherited Unix process context (such as signal
masks, stack size, CPU affinity). */
void restoreProcessContext();
masks, stack size). */
void restoreProcessContext(bool restoreMounts = true);
/* Save the current mount namespace. Ignored if called more than
once. */
void saveMountNamespace();
/* Restore the mount namespace saved by saveMountNamespace(). Ignored
if saveMountNamespace() was never called. */
void restoreMountNamespace();
/* Cause this thread to not share any FS attributes with the main
thread, because this causes setns() in restoreMountNamespace() to
fail. */
void unshareFilesystem();
class ExecError : public Error
@ -321,7 +343,7 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss);
/* Close all file descriptors except those listed in the given set.
Good practice in child processes. */
void closeMostFDs(const set<int> & exceptions);
void closeMostFDs(const std::set<int> & exceptions);
/* Set the close-on-exec flag for the given file descriptor. */
void closeOnExec(int fd);
@ -329,7 +351,7 @@ void closeOnExec(int fd);
/* User interruption. */
extern bool _isInterrupted;
extern std::atomic<bool> _isInterrupted;
extern thread_local std::function<bool()> interruptCheck;
@ -350,15 +372,19 @@ MakeError(FormatError, Error);
/* String tokenizer. */
template<class C> C tokenizeString(std::string_view s, const string & separators = " \t\n\r");
template<class C> C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r");
/* Concatenate the given strings with a separator between the
elements. */
template<class C>
string concatStringsSep(const string & sep, const C & ss)
std::string concatStringsSep(const std::string_view sep, const C & ss)
{
string s;
size_t size = 0;
// need a cast to string_view since this is also called with Symbols
for (const auto & s : ss) size += sep.size() + std::string_view(s).size();
std::string s;
s.reserve(size);
for (auto & i : ss) {
if (s.size() != 0) s += sep;
s += i;
@ -366,6 +392,14 @@ string concatStringsSep(const string & sep, const C & ss)
return s;
}
template<class ... Parts>
auto concatStrings(Parts && ... parts)
-> std::enable_if_t<(... && std::is_convertible_v<Parts, std::string_view>), std::string>
{
std::string_view views[sizeof...(parts)] = { parts... };
return concatStringsSep({}, views);
}
/* Add quotes around a collection of strings. */
template<class C> Strings quoteStrings(const C & c)
@ -379,45 +413,47 @@ template<class C> Strings quoteStrings(const C & c)
/* Remove trailing whitespace from a string. FIXME: return
std::string_view. */
string chomp(std::string_view s);
std::string chomp(std::string_view s);
/* Remove whitespace from the start and end of a string. */
string trim(const string & s, const string & whitespace = " \n\r\t");
std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t");
/* Replace all occurrences of a string inside another string. */
string replaceStrings(std::string_view s,
const std::string & from, const std::string & to);
std::string replaceStrings(
std::string s,
std::string_view from,
std::string_view to);
std::string rewriteStrings(const std::string & s, const StringMap & rewrites);
std::string rewriteStrings(std::string s, const StringMap & rewrites);
/* Convert the exit status of a child as returned by wait() into an
error string. */
string statusToString(int status);
std::string statusToString(int status);
bool statusOk(int status);
/* Parse a string into an integer. */
template<class N>
std::optional<N> string2Int(const std::string & s)
std::optional<N> string2Int(const std::string_view s)
{
if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
return std::nullopt;
std::istringstream str(s);
N n;
str >> n;
if (str && str.get() == EOF) return n;
return std::nullopt;
try {
return boost::lexical_cast<N>(s.data(), s.size());
} catch (const boost::bad_lexical_cast &) {
return std::nullopt;
}
}
/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or
'T' denoting a binary unit prefix. */
template<class N>
N string2IntWithUnitPrefix(std::string s)
N string2IntWithUnitPrefix(std::string_view s)
{
N multiplier = 1;
if (!s.empty()) {
@ -428,7 +464,7 @@ N string2IntWithUnitPrefix(std::string s)
else if (u == 'G') multiplier = 1ULL << 30;
else if (u == 'T') multiplier = 1ULL << 40;
else throw UsageError("invalid unit specifier '%1%'", u);
s.resize(s.size() - 1);
s.remove_suffix(1);
}
}
if (auto n = string2Int<N>(s))
@ -438,13 +474,13 @@ N string2IntWithUnitPrefix(std::string s)
/* Parse a string into a float. */
template<class N>
std::optional<N> string2Float(const string & s)
std::optional<N> string2Float(const std::string_view s)
{
std::istringstream str(s);
N n;
str >> n;
if (str && str.get() == EOF) return n;
return std::nullopt;
try {
return boost::lexical_cast<N>(s.data(), s.size());
} catch (const boost::bad_lexical_cast &) {
return std::nullopt;
}
}
@ -461,7 +497,7 @@ std::string toLower(const std::string & s);
/* Escape a string as a shell word. */
std::string shellEscape(const std::string & s);
std::string shellEscape(const std::string_view s);
/* Exception handling in destructors: print an error message, then
@ -491,8 +527,8 @@ std::string filterANSIEscapes(const std::string & s,
/* Base64 encoding/decoding. */
string base64Encode(std::string_view s);
string base64Decode(std::string_view s);
std::string base64Encode(std::string_view s);
std::string base64Decode(std::string_view s);
/* Remove common leading whitespace from the lines in the string
@ -511,6 +547,29 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_
}
/* Remove and return the first item from a container. */
template <class T>
std::optional<typename T::value_type> remove_begin(T & c)
{
auto i = c.begin();
if (i == c.end()) return {};
auto v = std::move(*i);
c.erase(i);
return v;
}
/* Remove and return the first item from a container. */
template <class T>
std::optional<typename T::value_type> pop(T & c)
{
if (c.empty()) return {};
auto v = std::move(c.front());
c.pop();
return v;
}
template<typename T>
class Callback;
@ -571,9 +630,18 @@ extern PathFilter defaultPathFilter;
/* Common initialisation performed in child processes. */
void commonChildInit(Pipe & logPipe);
/* Create a Unix domain socket. */
AutoCloseFD createUnixDomainSocket();
/* Create a Unix domain socket in listen mode. */
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
/* Bind a Unix domain socket to a path. */
void bind(int fd, const std::string & path);
/* Connect to a Unix domain socket. */
void connect(int fd, const std::string & path);
// A Rust/Python-like enumerate() iterator adapter.
// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17.

10
src/libutil/xml-writer.cc
View file

@ -31,11 +31,12 @@ void XMLWriter::close()
void XMLWriter::indent_(size_t depth)
{
if (!indent) return;
output << string(depth * 2, ' ');
output << std::string(depth * 2, ' ');
}
void XMLWriter::openElement(const string & name,
void XMLWriter::openElement(
std::string_view name,
const XMLAttrs & attrs)
{
assert(!closed);
@ -44,7 +45,7 @@ void XMLWriter::openElement(const string & name,
writeAttrs(attrs);
output << ">";
if (indent) output << std::endl;
pendingElems.push_back(name);
pendingElems.push_back(std::string(name));
}
@ -59,7 +60,8 @@ void XMLWriter::closeElement()
}
void XMLWriter::writeEmptyElement(const string & name,
void XMLWriter::writeEmptyElement(
std::string_view name,
const XMLAttrs & attrs)
{
assert(!closed);

14
src/libutil/xml-writer.hh
View file

@ -8,12 +8,8 @@
namespace nix {
using std::string;
using std::map;
using std::list;
typedef map<string, string> XMLAttrs;
typedef std::map<std::string, std::string> XMLAttrs;
class XMLWriter
@ -25,7 +21,7 @@ private:
bool indent;
bool closed;
list<string> pendingElems;
std::list<std::string> pendingElems;
public:
@ -34,11 +30,11 @@ public:
void close();
void openElement(const string & name,
void openElement(std::string_view name,
const XMLAttrs & attrs = XMLAttrs());
void closeElement();
void writeEmptyElement(const string & name,
void writeEmptyElement(std::string_view name,
const XMLAttrs & attrs = XMLAttrs());
private:
@ -53,7 +49,7 @@ class XMLOpenElement
private:
XMLWriter & writer;
public:
XMLOpenElement(XMLWriter & writer, const string & name,
XMLOpenElement(XMLWriter & writer, std::string_view name,
const XMLAttrs & attrs = XMLAttrs())
: writer(writer)
{