nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-24 03:09:35 +01:00
Code Activity

Merge remote-tracking branch 'origin/master' into progress-bar

Browse source
This commit is contained in:
Eelco Dolstra 2021-11-03 14:01:55 +01:00
commit c4f0508ef5
417 changed files with 42697 additions and 9071 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/libutil/ansicolor.hh
View file

@ -9,7 +9,7 @@ namespace nix {
#define ANSI_ITALIC "\e[3m"
#define ANSI_RED "\e[31;1m"
#define ANSI_GREEN "\e[32;1m"
#define ANSI_YELLOW "\e[33;1m"
#define ANSI_WARNING "\e[35;1m"
#define ANSI_BLUE "\e[34;1m"
#define ANSI_MAGENTA "\e[35;1m"
#define ANSI_CYAN "\e[36;1m"

4
src/libutil/archive.cc
View file

@ -42,7 +42,7 @@ static string caseHackSuffix = "~nix~case~hack~";
PathFilter defaultPathFilter = [](const Path &) { return true; };
static void dumpContents(const Path & path, size_t size,
static void dumpContents(const Path & path, off_t size,
Sink & sink)
{
sink << "contents" << size;
@ -76,7 +76,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
sink << "type" << "regular";
if (st.st_mode & S_IXUSR)
sink << "executable" << "";
dumpContents(path, (size_t) st.st_size, sink);
dumpContents(path, st.st_size, sink);
}
else if (S_ISDIR(st.st_mode)) {

119
src/libutil/args.cc
View file

@ -14,9 +14,19 @@ void Args::addFlag(Flag && flag_)
assert(flag->handler.arity == flag->labels.size());
assert(flag->longName != "");
longFlags[flag->longName] = flag;
for (auto & alias : flag->aliases)
longFlags[alias] = flag;
if (flag->shortName) shortFlags[flag->shortName] = flag;
}
void Args::removeFlag(const std::string & longName)
{
auto flag = longFlags.find(longName);
assert(flag != longFlags.end());
if (flag->second->shortName) shortFlags.erase(flag->second->shortName);
longFlags.erase(flag);
}
void Completions::add(std::string completion, std::string description)
{
assert(description.find('\n') == std::string::npos);
@ -58,6 +68,7 @@ void Args::parseCmdline(const Strings & _cmdline)
verbosity = lvlError;
}
bool argsSeen = false;
for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
auto arg = *pos;
@ -86,6 +97,10 @@ void Args::parseCmdline(const Strings & _cmdline)
throw UsageError("unrecognised flag '%1%'", arg);
}
else {
if (!argsSeen) {
argsSeen = true;
initialFlagsProcessed();
}
pos = rewriteArgs(cmdline, pos);
pendingArgs.push_back(*pos++);
if (processArgs(pendingArgs, false))
@ -94,41 +109,9 @@ void Args::parseCmdline(const Strings & _cmdline)
}
processArgs(pendingArgs, true);
}
void Args::printHelp(const string & programName, std::ostream & out)
{
std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName);
for (auto & exp : expectedArgs) {
std::cout << renderLabels({exp.label});
// FIXME: handle arity > 1
if (exp.handler.arity == ArityAny) std::cout << "...";
if (exp.optional) std::cout << "?";
}
std::cout << "\n";
auto s = description();
if (s != "")
std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n";
if (longFlags.size()) {
std::cout << "\n";
std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n";
printFlags(out);
}
}
void Args::printFlags(std::ostream & out)
{
Table2 table;
for (auto & flag : longFlags) {
if (hiddenCategories.count(flag.second->category)) continue;
table.push_back(std::make_pair(
(flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ")
+ "--" + flag.first + renderLabels(flag.second->labels),
flag.second->description));
}
printTable(out, table);
if (!argsSeen)
initialFlagsProcessed();
}
bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
@ -226,12 +209,12 @@ nlohmann::json Args::toJSON()
for (auto & [name, flag] : longFlags) {
auto j = nlohmann::json::object();
if (flag->aliases.count(name)) continue;
if (flag->shortName)
j["shortName"] = std::string(1, flag->shortName);
if (flag->description != "")
j["description"] = flag->description;
if (flag->category != "")
j["category"] = flag->category;
j["category"] = flag->category;
if (flag->handler.arity != ArityAny)
j["arity"] = flag->handler.arity;
if (!flag->labels.empty())
@ -331,30 +314,8 @@ Strings argvToStrings(int argc, char * * argv)
return args;
}
std::string renderLabels(const Strings & labels)
{
std::string res;
for (auto label : labels) {
for (auto & c : label) c = std::toupper(c);
res += " " ANSI_ITALIC + label + ANSI_NORMAL;
}
return res;
}
void printTable(std::ostream & out, const Table2 & table)
{
size_t max = 0;
for (auto & row : table)
max = std::max(max, filterANSIEscapes(row.first, true).size());
for (auto & row : table) {
out << " " << row.first
<< std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ')
<< row.second << "\n";
}
}
MultiCommand::MultiCommand(const Commands & commands)
: commands(commands)
MultiCommand::MultiCommand(const Commands & commands_)
: commands(commands_)
{
expectArgs({
.label = "subcommand",
@ -370,44 +331,13 @@ MultiCommand::MultiCommand(const Commands & commands)
if (i == commands.end())
throw UsageError("'%s' is not a recognised command", s);
command = {s, i->second()};
command->second->parent = this;
}}
});
categories[Command::catDefault] = "Available commands";
}
void MultiCommand::printHelp(const string & programName, std::ostream & out)
{
if (command) {
command->second->printHelp(programName + " " + command->first, out);
return;
}
out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName);
out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n";
printFlags(out);
std::map<Command::Category, std::map<std::string, ref<Command>>> commandsByCategory;
for (auto & [name, commandFun] : commands) {
auto command = commandFun();
commandsByCategory[command->category()].insert_or_assign(name, command);
}
for (auto & [category, commands] : commandsByCategory) {
out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]);
Table2 table;
for (auto & [name, command] : commands) {
auto descr = command->description();
if (!descr.empty())
table.push_back(std::make_pair(name, descr));
}
printTable(out, table);
}
}
bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)
{
if (Args::processFlag(pos, end)) return true;
@ -430,7 +360,10 @@ nlohmann::json MultiCommand::toJSON()
for (auto & [name, commandFun] : commands) {
auto command = commandFun();
auto j = command->toJSON();
j["category"] = categories[command->category()];
auto cat = nlohmann::json::object();
cat["id"] = command->category();
cat["description"] = categories[command->category()];
j["category"] = std::move(cat);
cmds[name] = std::move(j);
}

114
src/libutil/args.hh
View file

@ -12,6 +12,8 @@ namespace nix {
enum HashType : char;
class MultiCommand;
class Args
{
public:
@ -20,8 +22,6 @@ public:
wrong. */
void parseCmdline(const Strings & cmdline);
virtual void printHelp(const string & programName, std::ostream & out);
/* Return a short one-line description of the command. */
virtual std::string description() { return ""; }
@ -68,8 +68,12 @@ protected:
, arity(ArityAny)
{ }
template<class T>
Handler(T * dest)
Handler(std::string * dest)
: fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
, arity(1)
{ }
Handler(std::optional<std::string> * dest)
: fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
, arity(1)
{ }
@ -79,14 +83,31 @@ protected:
: fun([=](std::vector<std::string> ss) { *dest = val; })
, arity(0)
{ }
template<class I>
Handler(I * dest)
: fun([=](std::vector<std::string> ss) {
*dest = string2IntWithUnitPrefix<I>(ss[0]);
})
, arity(1)
{ }
template<class I>
Handler(std::optional<I> * dest)
: fun([=](std::vector<std::string> ss) {
*dest = string2IntWithUnitPrefix<I>(ss[0]);
})
, arity(1)
{ }
};
/* Flags. */
/* Options. */
struct Flag
{
typedef std::shared_ptr<Flag> ptr;
std::string longName;
std::set<std::string> aliases;
char shortName = 0;
std::string description;
std::string category;
@ -103,8 +124,6 @@ protected:
virtual bool processFlag(Strings::iterator & pos, Strings::iterator end);
virtual void printFlags(std::ostream & out);
/* Positional arguments. */
struct ExpectedArg
{
@ -123,70 +142,15 @@ protected:
std::set<std::string> hiddenCategories;
/* Called after all command line flags before the first non-flag
argument (if any) have been processed. */
virtual void initialFlagsProcessed() {}
public:
void addFlag(Flag && flag);
/* Helper functions for constructing flags / positional
arguments. */
void mkFlag1(char shortName, const std::string & longName,
const std::string & label, const std::string & description,
std::function<void(std::string)> fun)
{
addFlag({
.longName = longName,
.shortName = shortName,
.description = description,
.labels = {label},
.handler = {[=](std::string s) { fun(s); }}
});
}
void mkFlag(char shortName, const std::string & name,
const std::string & description, bool * dest)
{
mkFlag(shortName, name, description, dest, true);
}
template<class T>
void mkFlag(char shortName, const std::string & longName, const std::string & description,
T * dest, const T & value)
{
addFlag({
.longName = longName,
.shortName = shortName,
.description = description,
.handler = {[=]() { *dest = value; }}
});
}
template<class I>
void mkIntFlag(char shortName, const std::string & longName,
const std::string & description, I * dest)
{
mkFlag<I>(shortName, longName, description, [=](I n) {
*dest = n;
});
}
template<class I>
void mkFlag(char shortName, const std::string & longName,
const std::string & description, std::function<void(I)> fun)
{
addFlag({
.longName = longName,
.shortName = shortName,
.description = description,
.labels = {"N"},
.handler = {[=](std::string s) {
I n;
if (!string2Int(s, n))
throw UsageError("flag '--%s' requires a integer argument", longName);
fun(n);
}}
});
}
void removeFlag(const std::string & longName);
void expectArgs(ExpectedArg && arg)
{
@ -215,11 +179,13 @@ public:
virtual nlohmann::json toJSON();
friend class MultiCommand;
MultiCommand * parent = nullptr;
};
/* A command is an argument parser that can be executed by calling its
run() method. */
struct Command : virtual Args
struct Command : virtual public Args
{
friend class MultiCommand;
@ -239,7 +205,7 @@ typedef std::map<std::string, std::function<ref<Command>()>> Commands;
/* An argument parser that supports multiple subcommands,
i.e. ‘<command> <subcommand>’. */
class MultiCommand : virtual Args
class MultiCommand : virtual public Args
{
public:
Commands commands;
@ -251,8 +217,6 @@ public:
MultiCommand(const Commands & commands);
void printHelp(const string & programName, std::ostream & out) override;
bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
bool processArgs(const Strings & args, bool finish) override;
@ -262,14 +226,6 @@ public:
Strings argvToStrings(int argc, char * * argv);
/* Helper function for rendering argument labels. */
std::string renderLabels(const Strings & labels);
/* Helper function for printing 2-column tables. */
typedef std::vector<std::pair<std::string, std::string>> Table2;
void printTable(std::ostream & out, const Table2 & table);
struct Completion {
std::string completion;
std::string description;

69
src/libutil/closure.hh Normal file
View file

@ -0,0 +1,69 @@
#include <set>
#include <future>
#include "sync.hh"
using std::set;
namespace nix {
template<typename T>
using GetEdgesAsync = std::function<void(const T &, std::function<void(std::promise<set<T>> &)>)>;
template<typename T>
void computeClosure(
const set<T> startElts,
set<T> & res,
GetEdgesAsync<T> getEdgesAsync
)
{
struct State
{
size_t pending;
set<T> & res;
std::exception_ptr exc;
};
Sync<State> state_(State{0, res, 0});
std::function<void(const T &)> enqueue;
std::condition_variable done;
enqueue = [&](const T & current) -> void {
{
auto state(state_.lock());
if (state->exc) return;
if (!state->res.insert(current).second) return;
state->pending++;
}
getEdgesAsync(current, [&](std::promise<set<T>> & prom) {
try {
auto children = prom.get_future().get();
for (auto & child : children)
enqueue(child);
{
auto state(state_.lock());
assert(state->pending);
if (!--state->pending) done.notify_one();
}
} catch (...) {
auto state(state_.lock());
if (!state->exc) state->exc = std::current_exception();
assert(state->pending);
if (!--state->pending) done.notify_one();
};
});
};
for (auto & startElt : startElts)
enqueue(startElt);
{
auto state(state_.lock());
while (state->pending) state.wait(done);
if (state->exc) std::rethrow_exception(state->exc);
}
}
}

32
src/libutil/comparator.hh Normal file
View file

@ -0,0 +1,32 @@
#pragma once
/* Awfull hacky generation of the comparison operators by doing a lexicographic
* comparison between the choosen fields.
*
* ```
* GENERATE_CMP(ClassName, me->field1, me->field2, ...)
* ```
*
* will generate comparison operators semantically equivalent to:
*
* ```
* bool operator<(const ClassName& other) {
* return field1 < other.field1 && field2 < other.field2 && ...;
* }
* ```
*/
#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \
bool operator COMPARATOR(const MY_TYPE& other) const { \
const MY_TYPE* me = this; \
auto fields1 = std::make_tuple( FIELDS ); \
me = &other; \
auto fields2 = std::make_tuple( FIELDS ); \
return fields1 COMPARATOR fields2; \
}
#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
#define GENERATE_CMP(args...) \
GENERATE_EQUAL(args) \
GENERATE_LEQ(args) \
GENERATE_NEQ(args)

404
src/libutil/compression.cc
View file

@ -1,22 +1,23 @@
#include "compression.hh"
#include "tarfile.hh"
#include "util.hh"
#include "finally.hh"
#include "logging.hh"
#include <lzma.h>
#include <bzlib.h>
#include <archive.h>
#include <archive_entry.h>
#include <cstdio>
#include <cstring>
#include <brotli/decode.h>
#include <brotli/encode.h>
#include <zlib.h>
#include <iostream>
namespace nix {
static const int COMPRESSION_LEVEL_DEFAULT = -1;
// Don't feed brotli too much at once.
struct ChunkedCompressionSink : CompressionSink
{
@ -27,7 +28,7 @@ struct ChunkedCompressionSink : CompressionSink
const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
while (!data.empty()) {
size_t n = std::min(CHUNK_SIZE, data.size());
writeInternal(data);
writeInternal(data.substr(0, n));
data.remove_prefix(n);
}
}
@ -35,177 +36,107 @@ struct ChunkedCompressionSink : CompressionSink
virtual void writeInternal(std::string_view data) = 0;
};
struct NoneSink : CompressionSink
struct ArchiveDecompressionSource : Source
{
Sink & nextSink;
NoneSink(Sink & nextSink) : nextSink(nextSink) { }
void finish() override { flush(); }
void write(std::string_view data) override { nextSink(data); }
};
struct GzipDecompressionSink : CompressionSink
{
Sink & nextSink;
z_stream strm;
bool finished = false;
uint8_t outbuf[BUFSIZ];
GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
// Enable gzip and zlib decoding (+32) with 15 windowBits
int ret = inflateInit2(&strm,15+32);
if (ret != Z_OK)
throw CompressionError("unable to initialise gzip encoder");
}
~GzipDecompressionSink()
{
inflateEnd(&strm);
}
void finish() override
{
CompressionSink::flush();
write({});
}
void write(std::string_view data) override
{
assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (Bytef *) data.data();
strm.avail_in = data.size();
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
int ret = inflate(&strm,Z_SYNC_FLUSH);
if (ret != Z_OK && ret != Z_STREAM_END)
throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
zError(ret), data.size(), strm.avail_in);
finished = ret == Z_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (Bytef *) outbuf;
strm.avail_out = sizeof(outbuf);
std::unique_ptr<TarArchive> archive = 0;
Source & src;
ArchiveDecompressionSource(Source & src) : src(src) {}
~ArchiveDecompressionSource() override {}
size_t read(char * data, size_t len) override {
struct archive_entry * ae;
if (!archive) {
archive = std::make_unique<TarArchive>(src, true);
this->archive->check(archive_read_next_header(this->archive->archive, &ae),
"failed to read header (%s)");
if (archive_filter_count(this->archive->archive) < 2) {
throw CompressionError("input compression not recognized");
}
}
}
};
struct XzDecompressionSink : CompressionSink
{
Sink & nextSink;
uint8_t outbuf[BUFSIZ];
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
XzDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
lzma_ret ret = lzma_stream_decoder(
&strm, UINT64_MAX, LZMA_CONCATENATED);
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma decoder");
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
~XzDecompressionSink()
{
lzma_end(&strm);
}
void finish() override
{
CompressionSink::flush();
write({});
}
void write(std::string_view data) override
{
strm.next_in = (const unsigned char *) data.data();
strm.avail_in = data.size();
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while decompressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
ssize_t result = archive_read_data(this->archive->archive, data, len);
if (result > 0) return result;
if (result == 0) {
throw EndOfFile("reached end of compressed file");
}
this->archive->check(result, "failed to read compressed data (%s)");
return result;
}
};
struct BzipDecompressionSink : ChunkedCompressionSink
struct ArchiveCompressionSink : CompressionSink
{
Sink & nextSink;
bz_stream strm;
bool finished = false;
struct archive * archive;
BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink)
{
memset(&strm, 0, sizeof(strm));
int ret = BZ2_bzDecompressInit(&strm, 0, 0);
if (ret != BZ_OK)
throw CompressionError("unable to initialise bzip2 decoder");
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
archive = archive_write_new();
if (!archive) throw Error("failed to initialize libarchive");
check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
check(archive_write_set_format_raw(archive));
if (parallel)
check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
if (level != COMPRESSION_LEVEL_DEFAULT)
check(archive_write_set_filter_option(archive, format.c_str(), "compression-level", std::to_string(level).c_str()));
// disable internal buffering
check(archive_write_set_bytes_per_block(archive, 0));
// disable output padding
check(archive_write_set_bytes_in_last_block(archive, 1));
open();
}
~BzipDecompressionSink()
~ArchiveCompressionSink() override
{
BZ2_bzDecompressEnd(&strm);
if (archive) archive_write_free(archive);
}
void finish() override
{
flush();
write({});
check(archive_write_close(archive));
}
void writeInternal(std::string_view data) override
void check(int err, const std::string & reason = "failed to compress (%s)")
{
assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char *) data.data();
strm.avail_in = data.size();
while (strm.avail_in) {
checkInterrupt();
int ret = BZ2_bzDecompress(&strm);
if (ret != BZ_OK && ret != BZ_STREAM_END)
throw CompressionError("error while decompressing bzip2 file");
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
}
if (err == ARCHIVE_EOF)
throw EndOfFile("reached end of archive");
else if (err != ARCHIVE_OK)
throw Error(reason, archive_error_string(this->archive));
}
void write(std::string_view data) override
{
ssize_t result = archive_write_data(archive, data.data(), data.length());
if (result <= 0) check(result);
}
private:
void open()
{
check(archive_write_open(archive, this, nullptr, ArchiveCompressionSink::callback_write, nullptr));
auto ae = archive_entry_new();
archive_entry_set_filetype(ae, AE_IFREG);
check(archive_write_header(archive, ae));
archive_entry_free(ae);
}
static ssize_t callback_write(struct archive * archive, void * _self, const void * buffer, size_t length)
{
auto self = (ArchiveCompressionSink *) _self;
self->nextSink({(const char *) buffer, length});
return length;
}
};
struct NoneSink : CompressionSink
{
Sink & nextSink;
NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink)
{
if (level != COMPRESSION_LEVEL_DEFAULT)
warn("requested compression level '%d' not supported by compression method 'none'", level);
}
void finish() override { flush(); }
void write(std::string_view data) override { nextSink(data); }
};
struct BrotliDecompressionSink : ChunkedCompressionSink
@ -268,159 +199,24 @@ ref<std::string> decompress(const std::string & method, const std::string & in)
return ssink.s;
}
ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
{
if (method == "none" || method == "")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
return make_ref<XzDecompressionSink>(nextSink);
else if (method == "bzip2")
return make_ref<BzipDecompressionSink>(nextSink);
else if (method == "gzip")
return make_ref<GzipDecompressionSink>(nextSink);
return std::make_unique<NoneSink>(nextSink);
else if (method == "br")
return make_ref<BrotliDecompressionSink>(nextSink);
return std::make_unique<BrotliDecompressionSink>(nextSink);
else
throw UnknownCompressionMethod("unknown compression method '%s'", method);
return sourceToSink([&](Source & source) {
auto decompressionSource = std::make_unique<ArchiveDecompressionSource>(source);
decompressionSource->drainInto(nextSink);
});
}
struct XzCompressionSink : CompressionSink
{
Sink & nextSink;
uint8_t outbuf[BUFSIZ];
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink)
{
lzma_ret ret;
bool done = false;
if (parallel) {
#ifdef HAVE_LZMA_MT
lzma_mt mt_options = {};
mt_options.flags = 0;
mt_options.timeout = 300; // Using the same setting as the xz cmd line
mt_options.preset = LZMA_PRESET_DEFAULT;
mt_options.filters = NULL;
mt_options.check = LZMA_CHECK_CRC64;
mt_options.threads = lzma_cputhreads();
mt_options.block_size = 0;
if (mt_options.threads == 0)
mt_options.threads = 1;
// FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
// number of threads.
ret = lzma_stream_encoder_mt(&strm, &mt_options);
done = true;
#else
printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
#endif
}
if (!done)
ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma encoder");
// FIXME: apply the x86 BCJ filter?
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
~XzCompressionSink()
{
lzma_end(&strm);
}
void finish() override
{
CompressionSink::flush();
write({});
}
void write(std::string_view data) override
{
strm.next_in = (const unsigned char *) data.data();
strm.avail_in = data.size();
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while compressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
};
struct BzipCompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
bz_stream strm;
bool finished = false;
BzipCompressionSink(Sink & nextSink) : nextSink(nextSink)
{
memset(&strm, 0, sizeof(strm));
int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
if (ret != BZ_OK)
throw CompressionError("unable to initialise bzip2 encoder");
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
~BzipCompressionSink()
{
BZ2_bzCompressEnd(&strm);
}
void finish() override
{
flush();
writeInternal({});
}
void writeInternal(std::string_view data) override
{
assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char *) data.data();
strm.avail_in = data.size();
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH);
if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
throw CompressionError("error %d while compressing bzip2 file", ret);
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
};
struct BrotliCompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
uint8_t outbuf[BUFSIZ];
BrotliEncoderState *state;
BrotliEncoderState * state;
bool finished = false;
BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink)
@ -469,24 +265,26 @@ struct BrotliCompressionSink : ChunkedCompressionSink
}
};
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
{
std::vector<std::string> la_supports = {
"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"
};
if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
}
if (method == "none")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
return make_ref<XzCompressionSink>(nextSink, parallel);
else if (method == "bzip2")
return make_ref<BzipCompressionSink>(nextSink);
else if (method == "br")
return make_ref<BrotliCompressionSink>(nextSink);
else
throw UnknownCompressionMethod("unknown compression method '%s'", method);
}
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel, int level)
{
StringSink ssink;
auto sink = makeCompressionSink(method, ssink, parallel);
auto sink = makeCompressionSink(method, ssink, parallel, level);
(*sink)(in);
sink->finish();
return ssink.s;

12
src/libutil/compression.hh
View file

@ -8,18 +8,20 @@
namespace nix {
struct CompressionSink : BufferedSink
struct CompressionSink : BufferedSink, FinishSink
{
virtual void finish() = 0;
using BufferedSink::operator ();
using BufferedSink::write;
using FinishSink::finish;
};
ref<std::string> decompress(const std::string & method, const std::string & in);
ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false, int level = -1);
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
MakeError(UnknownCompressionMethod, Error);

80
src/libutil/compute-levels.cc Normal file
View file

@ -0,0 +1,80 @@
#include "types.hh"
#if HAVE_LIBCPUID
#include <libcpuid/libcpuid.h>
#endif
namespace nix {
#if HAVE_LIBCPUID
StringSet computeLevels() {
StringSet levels;
if (!cpuid_present())
return levels;
cpu_raw_data_t raw;
cpu_id_t data;
if (cpuid_get_raw_data(&raw) < 0)
return levels;
if (cpu_identify(&raw, &data) < 0)
return levels;
if (!(data.flags[CPU_FEATURE_CMOV] &&
data.flags[CPU_FEATURE_CX8] &&
data.flags[CPU_FEATURE_FPU] &&
data.flags[CPU_FEATURE_FXSR] &&
data.flags[CPU_FEATURE_MMX] &&
data.flags[CPU_FEATURE_SSE] &&
data.flags[CPU_FEATURE_SSE2]))
return levels;
levels.insert("x86_64-v1");
if (!(data.flags[CPU_FEATURE_CX16] &&
data.flags[CPU_FEATURE_LAHF_LM] &&
data.flags[CPU_FEATURE_POPCNT] &&
// SSE3
data.flags[CPU_FEATURE_PNI] &&
data.flags[CPU_FEATURE_SSSE3] &&
data.flags[CPU_FEATURE_SSE4_1] &&
data.flags[CPU_FEATURE_SSE4_2]))
return levels;
levels.insert("x86_64-v2");
if (!(data.flags[CPU_FEATURE_AVX] &&
data.flags[CPU_FEATURE_AVX2] &&
data.flags[CPU_FEATURE_F16C] &&
data.flags[CPU_FEATURE_FMA3] &&
// LZCNT
data.flags[CPU_FEATURE_ABM] &&
data.flags[CPU_FEATURE_MOVBE]))
return levels;
levels.insert("x86_64-v3");
if (!(data.flags[CPU_FEATURE_AVX512F] &&
data.flags[CPU_FEATURE_AVX512BW] &&
data.flags[CPU_FEATURE_AVX512CD] &&
data.flags[CPU_FEATURE_AVX512DQ] &&
data.flags[CPU_FEATURE_AVX512VL]))
return levels;
levels.insert("x86_64-v4");
return levels;
}
#else
StringSet computeLevels() {
return StringSet{};
}
#endif // HAVE_LIBCPUID
}

7
src/libutil/compute-levels.hh Normal file
View file

@ -0,0 +1,7 @@
#include "types.hh"
namespace nix {
StringSet computeLevels();
}

82
src/libutil/config.cc
View file

@ -1,6 +1,7 @@
#include "config.hh"
#include "args.hh"
#include "abstract-setting-to-json.hh"
#include "experimental-features.hh"
#include <nlohmann/json.hpp>
@ -20,7 +21,7 @@ bool Config::set(const std::string & name, const std::string & value)
return false;
}
i->second.setting->set(value, append);
i->second.setting->overriden = true;
i->second.setting->overridden = true;
return true;
}
@ -35,7 +36,7 @@ void Config::addSetting(AbstractSetting * setting)
auto i = unknownSettings.find(setting->name);
if (i != unknownSettings.end()) {
setting->set(i->second);
setting->overriden = true;
setting->overridden = true;
unknownSettings.erase(i);
set = true;
}
@ -48,7 +49,7 @@ void Config::addSetting(AbstractSetting * setting)
alias, setting->name);
else {
setting->set(i->second);
setting->overriden = true;
setting->overridden = true;
unknownSettings.erase(i);
set = true;
}
@ -69,10 +70,10 @@ void AbstractConfig::reapplyUnknownSettings()
set(s.first, s.second);
}
void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
{
for (auto & opt : _settings)
if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden))
if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
}
@ -136,10 +137,10 @@ void AbstractConfig::applyConfigFile(const Path & path)
} catch (SysError &) { }
}
void Config::resetOverriden()
void Config::resetOverridden()
{
for (auto & s : _settings)
s.second.setting->overriden = false;
s.second.setting->overridden = false;
}
nlohmann::json Config::toJSON()
@ -152,6 +153,16 @@ nlohmann::json Config::toJSON()
return res;
}
std::string Config::toKeyValue()
{
auto res = std::string();
for (auto & s : _settings)
if (!s.second.isAlias) {
res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
}
return res;
}
void Config::convertToArgs(Args & args, const std::string & category)
{
for (auto & s : _settings)
@ -167,11 +178,6 @@ AbstractSetting::AbstractSetting(
{
}
void AbstractSetting::setDefault(const std::string & str)
{
if (!overriden) set(str);
}
nlohmann::json AbstractSetting::toJSON()
{
return nlohmann::json(toJSONObject());
@ -203,7 +209,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
.description = fmt("Set the `%s` setting.", name),
.category = category,
.labels = {"value"},
.handler = {[=](std::string s) { overriden = true; set(s); }},
.handler = {[=](std::string s) { overridden = true; set(s); }},
});
if (isAppendable())
@ -212,7 +218,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
.description = fmt("Append to the `%s` setting.", name),
.category = category,
.labels = {"value"},
.handler = {[=](std::string s) { overriden = true; set(s, true); }},
.handler = {[=](std::string s) { overridden = true; set(s, true); }},
});
}
@ -230,7 +236,9 @@ template<typename T>
void BaseSetting<T>::set(const std::string & str, bool append)
{
static_assert(std::is_integral<T>::value, "Integer required.");
if (!string2Int(str, value))
if (auto n = string2Int<T>(str))
value = *n;
else
throw UsageError("setting '%s' has invalid value '%s'", name, str);
}
@ -306,6 +314,31 @@ template<> std::string BaseSetting<StringSet>::to_string() const
return concatStringsSep(" ", value);
}
template<> void BaseSetting<std::set<ExperimentalFeature>>::set(const std::string & str, bool append)
{
if (!append) value.clear();
for (auto & s : tokenizeString<StringSet>(str)) {
auto thisXpFeature = parseExperimentalFeature(s);
if (thisXpFeature)
value.insert(thisXpFeature.value());
else
warn("unknown experimental feature '%s'", s);
}
}
template<> bool BaseSetting<std::set<ExperimentalFeature>>::isAppendable()
{
return true;
}
template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() const
{
StringSet stringifiedXpFeatures;
for (auto & feature : value)
stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature)));
return concatStringsSep(" ", stringifiedXpFeatures);
}
template<> void BaseSetting<StringMap>::set(const std::string & str, bool append)
{
if (!append) value.clear();
@ -341,6 +374,7 @@ template class BaseSetting<std::string>;
template class BaseSetting<Strings>;
template class BaseSetting<StringSet>;
template class BaseSetting<StringMap>;
template class BaseSetting<std::set<ExperimentalFeature>>;
void PathSetting::set(const std::string & str, bool append)
{
@ -363,16 +397,16 @@ bool GlobalConfig::set(const std::string & name, const std::string & value)
return false;
}
void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
{
for (auto & config : *configRegistrations)
config->getSettings(res, overridenOnly);
config->getSettings(res, overriddenOnly);
}
void GlobalConfig::resetOverriden()
void GlobalConfig::resetOverridden()
{
for (auto & config : *configRegistrations)
config->resetOverriden();
config->resetOverridden();
}
nlohmann::json GlobalConfig::toJSON()
@ -383,6 +417,16 @@ nlohmann::json GlobalConfig::toJSON()
return res;
}
std::string GlobalConfig::toKeyValue()
{
std::string res;
std::map<std::string, Config::SettingInfo> settings;
globalConfig.getSettings(settings);
for (auto & s : settings)
res += fmt("%s = %s\n", s.first, s.second.value);
return res;
}
void GlobalConfig::convertToArgs(Args & args, const std::string & category)
{
for (auto & config : *configRegistrations)

35
src/libutil/config.hh
View file

@ -71,9 +71,9 @@ public:
/**
* Adds the currently known settings to the given result map `res`.
* - res: map to store settings in
* - overridenOnly: when set to true only overridden settings will be added to `res`
* - overriddenOnly: when set to true only overridden settings will be added to `res`
*/
virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0;
virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly = false) = 0;
/**
* Parses the configuration in `contents` and applies it
@ -91,7 +91,7 @@ public:
/**
* Resets the `overridden` flag of all Settings
*/
virtual void resetOverriden() = 0;
virtual void resetOverridden() = 0;
/**
* Outputs all settings to JSON
@ -99,6 +99,12 @@ public:
*/
virtual nlohmann::json toJSON() = 0;
/**
* Outputs all settings in a key-value pair format suitable to be used as
* `nix.conf`
*/
virtual std::string toKeyValue() = 0;
/**
* Converts settings to `Args` to be used on the command line interface
* - args: args to write to
@ -127,7 +133,7 @@ public:
MyClass() : Config(readConfigFile("/etc/my-app.conf"))
{
std::cout << foo << "\n"; // will print 123 unless overriden
std::cout << foo << "\n"; // will print 123 unless overridden
}
};
*/
@ -163,12 +169,14 @@ public:
void addSetting(AbstractSetting * setting);
void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
void getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly = false) override;
void resetOverriden() override;
void resetOverridden() override;
nlohmann::json toJSON() override;
std::string toKeyValue() override;
void convertToArgs(Args & args, const std::string & category) override;
};
@ -184,9 +192,7 @@ public:
int created = 123;
bool overriden = false;
void setDefault(const std::string & str);
bool overridden = false;
protected:
@ -215,7 +221,7 @@ protected:
virtual void convertToArg(Args & args, const std::string & category);
bool isOverriden() const { return overriden; }
bool isOverridden() const { return overridden; }
};
/* A setting of type T. */
@ -245,6 +251,7 @@ public:
bool operator !=(const T & v2) const { return value != v2; }
void operator =(const T & v) { assign(v); }
virtual void assign(const T & v) { value = v; }
void setDefault(const T & v) { if (!overridden) value = v; }
void set(const std::string & str, bool append = false) override;
@ -252,7 +259,7 @@ public:
virtual void override(const T & v)
{
overriden = true;
overridden = true;
value = v;
}
@ -324,12 +331,14 @@ struct GlobalConfig : public AbstractConfig
bool set(const std::string & name, const std::string & value) override;
void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
void getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly = false) override;
void resetOverriden() override;
void resetOverridden() override;
nlohmann::json toJSON() override;
std::string toKeyValue() override;
void convertToArgs(Args & args, const std::string & category) override;
struct Register

188
src/libutil/error.cc
View file

@ -43,9 +43,9 @@ string showErrPos(const ErrPos & errPos)
{
if (errPos.line > 0) {
if (errPos.column > 0) {
return fmt("(%1%:%2%)", errPos.line, errPos.column);
return fmt("%d:%d", errPos.line, errPos.column);
} else {
return fmt("(%1%)", errPos.line);
return fmt("%d", errPos.line);
}
}
else {
@ -61,6 +61,8 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
if (errPos.origin == foFile) {
LinesOfCode loc;
try {
// FIXME: when running as the daemon, make sure we don't
// open a file to which the client doesn't have access.
AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) return {};
@ -178,24 +180,20 @@ void printCodeLines(std::ostream & out,
}
}
void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out)
void printAtPos(const ErrPos & pos, std::ostream & out)
{
if (pos)
{
if (pos) {
switch (pos.origin) {
case foFile: {
out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file;
out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos));
break;
}
case foString: {
out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
ANSI_BLUE << " from string" << ANSI_NORMAL;
out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos));
break;
}
case foStdin: {
out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
ANSI_BLUE << " from stdin" << ANSI_NORMAL;
out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos));
break;
}
default:
@ -204,168 +202,108 @@ void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out)
}
}
static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s)
{
std::string res;
bool first = true;
while (!s.empty()) {
auto end = s.find('\n');
if (!first) res += "\n";
res += chomp(std::string(first ? indentFirst : indentRest) + std::string(s.substr(0, end)));
first = false;
if (end == s.npos) break;
s = s.substr(end + 1);
}
return res;
}
std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace)
{
auto errwidth = std::max<size_t>(getWindowSize().second, 20);
string prefix = "";
string levelString;
std::string prefix;
switch (einfo.level) {
case Verbosity::lvlError: {
levelString = ANSI_RED;
levelString += "error:";
levelString += ANSI_NORMAL;
prefix = ANSI_RED "error";
break;
}
case Verbosity::lvlNotice: {
prefix = ANSI_RED "note";
break;
}
case Verbosity::lvlWarn: {
levelString = ANSI_YELLOW;
levelString += "warning:";
levelString += ANSI_NORMAL;
prefix = ANSI_WARNING "warning";
break;
}
case Verbosity::lvlInfo: {
levelString = ANSI_GREEN;
levelString += "info:";
levelString += ANSI_NORMAL;
prefix = ANSI_GREEN "info";
break;
}
case Verbosity::lvlTalkative: {
levelString = ANSI_GREEN;
levelString += "talk:";
levelString += ANSI_NORMAL;
prefix = ANSI_GREEN "talk";
break;
}
case Verbosity::lvlChatty: {
levelString = ANSI_GREEN;
levelString += "chat:";
levelString += ANSI_NORMAL;
prefix = ANSI_GREEN "chat";
break;
}
case Verbosity::lvlVomit: {
levelString = ANSI_GREEN;
levelString += "vomit:";
levelString += ANSI_NORMAL;
prefix = ANSI_GREEN "vomit";
break;
}
case Verbosity::lvlDebug: {
levelString = ANSI_YELLOW;
levelString += "debug:";
levelString += ANSI_NORMAL;
break;
}
default: {
levelString = fmt("invalid error level: %1%", einfo.level);
prefix = ANSI_WARNING "debug";
break;
}
default:
assert(false);
}
auto ndl = prefix.length()
+ filterANSIEscapes(levelString, true).length()
+ 7
+ einfo.name.length()
+ einfo.programName.value_or("").length();
auto dashwidth = std::max<int>(errwidth - ndl, 3);
std::string dashes(dashwidth, '-');
// divider.
if (einfo.name != "")
out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL,
prefix,
levelString,
einfo.name,
dashes,
einfo.programName.value_or(""));
// FIXME: show the program name as part of the trace?
if (einfo.programName && einfo.programName != ErrorInfo::programName)
prefix += fmt(" [%s]:" ANSI_NORMAL " ", einfo.programName.value_or(""));
else
out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL,
prefix,
levelString,
dashes,
einfo.programName.value_or(""));
prefix += ":" ANSI_NORMAL " ";
bool nl = false; // intersperse newline between sections.
if (einfo.errPos.has_value() && (*einfo.errPos)) {
out << prefix << std::endl;
printAtPos(prefix, *einfo.errPos, out);
nl = true;
}
std::ostringstream oss;
oss << einfo.msg << "\n";
// description
if (einfo.description != "") {
if (nl)
out << std::endl << prefix;
out << std::endl << prefix << einfo.description;
nl = true;
}
if (einfo.errPos.has_value() && *einfo.errPos) {
oss << "\n";
printAtPos(*einfo.errPos, oss);
if (einfo.errPos.has_value() && (*einfo.errPos)) {
auto loc = getCodeLines(*einfo.errPos);
// lines of code.
if (loc.has_value()) {
if (nl)
out << std::endl << prefix;
printCodeLines(out, prefix, *einfo.errPos, *loc);
nl = true;
oss << "\n";
printCodeLines(oss, "", *einfo.errPos, *loc);
oss << "\n";
}
}
// hint
if (einfo.hint.has_value()) {
if (nl)
out << std::endl << prefix;
out << std::endl << prefix << *einfo.hint;
nl = true;
}
// traces
if (showTrace && !einfo.traces.empty())
{
const string tracetitle(" show-trace ");
int fill = errwidth - tracetitle.length();
int lw = 0;
int rw = 0;
const int min_dashes = 3;
if (fill > min_dashes * 2) {
if (fill % 2 != 0) {
lw = fill / 2;
rw = lw + 1;
}
else
{
lw = rw = fill / 2;
}
}
else
lw = rw = min_dashes;
if (nl)
out << std::endl << prefix;
out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL;
for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter)
{
out << std::endl << prefix;
out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str();
if (showTrace && !einfo.traces.empty()) {
for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) {
oss << "\n" << "… " << iter->hint.str() << "\n";
if (iter->pos.has_value() && (*iter->pos)) {
auto pos = iter->pos.value();
out << std::endl << prefix;
printAtPos(prefix, pos, out);
oss << "\n";
printAtPos(pos, oss);
auto loc = getCodeLines(pos);
if (loc.has_value())
{
out << std::endl << prefix;
printCodeLines(out, prefix, pos, *loc);
out << std::endl << prefix;
if (loc.has_value()) {
oss << "\n";
printCodeLines(oss, "", pos, *loc);
oss << "\n";
}
}
}
}
out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str()));
return out;
}
}

21
src/libutil/error.hh
View file

@ -38,7 +38,7 @@ namespace nix {
ErrorInfo structs are sent to the logger as part of an exception, or directly with the
logError or logWarning macros.
See the error-demo.cc program for usage examples.
See libutil/tests/logging.cc for usage examples.
*/
@ -107,9 +107,8 @@ struct Trace {
struct ErrorInfo {
Verbosity level;
string name;
string description; // FIXME: remove? it seems to be barely used
std::optional<hintformat> hint;
string name; // FIXME: rename
hintformat msg;
std::optional<ErrPos> errPos;
std::list<Trace> traces;
@ -133,23 +132,17 @@ public:
template<typename... Args>
BaseError(unsigned int status, const Args & ... args)
: err {.level = lvlError,
.hint = hintfmt(args...)
}
: err { .level = lvlError, .msg = hintfmt(args...) }
, status(status)
{ }
template<typename... Args>
BaseError(const std::string & fs, const Args & ... args)
: err {.level = lvlError,
.hint = hintfmt(fs, args...)
}
: err { .level = lvlError, .msg = hintfmt(fs, args...) }
{ }
BaseError(hintformat hint)
: err {.level = lvlError,
.hint = hint
}
: err { .level = lvlError, .msg = hint }
{ }
BaseError(ErrorInfo && e)
@ -206,7 +199,7 @@ public:
{
errNo = errno;
auto hf = hintfmt(args...);
err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
}
virtual const char* sname() const override { return "SysError"; }

59
src/libutil/experimental-features.cc Normal file
View file

@ -0,0 +1,59 @@
#include "experimental-features.hh"
#include "util.hh"
#include "nlohmann/json.hpp"
namespace nix {
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::CaDerivations, "ca-derivations" },
{ Xp::Flakes, "flakes" },
{ Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" },
{ Xp::NoUrlLiterals, "no-url-literals" },
};
const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
{
using ReverseXpMap = std::map<std::string_view, ExperimentalFeature>;
static auto reverseXpMap = []()
{
auto reverseXpMap = std::make_unique<ReverseXpMap>();
for (auto & [feature, name] : stringifiedXpFeatures)
(*reverseXpMap)[name] = feature;
return reverseXpMap;
}();
if (auto feature = get(*reverseXpMap, name))
return *feature;
else
return std::nullopt;
}
std::string_view showExperimentalFeature(const ExperimentalFeature feature)
{
return stringifiedXpFeatures.at(feature);
}
std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
{
std::set<ExperimentalFeature> res;
for (auto & rawFeature : rawFeatures) {
if (auto feature = parseExperimentalFeature(rawFeature))
res.insert(*feature);
}
return res;
}
MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature)
: Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature))
, missingFeature(feature)
{}
std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & feature)
{
return str << showExperimentalFeature(feature);
}
}

56
src/libutil/experimental-features.hh Normal file
View file

@ -0,0 +1,56 @@
#pragma once
#include "comparator.hh"
#include "error.hh"
#include "nlohmann/json_fwd.hpp"
#include "types.hh"
namespace nix {
/**
* The list of available experimental features.
*
* If you update this, don’t forget to also change the map defining their
* string representation in the corresponding `.cc` file.
**/
enum struct ExperimentalFeature
{
CaDerivations,
Flakes,
NixCommand,
RecursiveNix,
NoUrlLiterals
};
/**
* Just because writing `ExperimentalFeature::CaDerivations` is way too long
*/
using Xp = ExperimentalFeature;
const std::optional<ExperimentalFeature> parseExperimentalFeature(
const std::string_view & name);
std::string_view showExperimentalFeature(const ExperimentalFeature);
std::ostream & operator<<(
std::ostream & str,
const ExperimentalFeature & feature);
/**
* Parse a set of strings to the corresponding set of experimental features,
* ignoring (but warning for) any unkwown feature.
*/
std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> &);
class MissingExperimentalFeature : public Error
{
public:
ExperimentalFeature missingFeature;
MissingExperimentalFeature(ExperimentalFeature);
virtual const char * sname() const override
{
return "MissingExperimentalFeature";
}
};
}

2
src/libutil/fmt.hh
View file

@ -82,7 +82,7 @@ struct yellowtxt
template <class T>
std::ostream & operator<<(std::ostream & out, const yellowtxt<T> & y)
{
return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
return out << ANSI_WARNING << y.value << ANSI_NORMAL;
}
template <class T>

6
src/libutil/local.mk
View file

@ -6,4 +6,8 @@ libutil_DIR := $(d)
libutil_SOURCES := $(wildcard $(d)/*.cc)
libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
ifeq ($(HAVE_LIBCPUID), 1)
libutil_LDFLAGS += -lcpuid
endif

13
src/libutil/logging.cc
View file

@ -27,7 +27,7 @@ Logger * logger = makeSimpleLogger(true);
void Logger::warn(const std::string & msg)
{
log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg);
log(lvlWarn, ANSI_WARNING "warning:" ANSI_NORMAL " " + msg);
}
void Logger::writeToStdout(std::string_view s)
@ -46,7 +46,7 @@ public:
: printBuildLogs(printBuildLogs)
{
systemd = getEnv("IN_SYSTEMD") == "1";
tty = isatty(STDERR_FILENO);
tty = shouldANSI();
}
bool isVerbose() override {
@ -163,7 +163,7 @@ struct JSONLogger : Logger {
void write(const nlohmann::json & json)
{
prevLogger.log(lvlError, "@nix " + json.dump());
prevLogger.log(lvlError, "@nix " + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace));
}
void log(Verbosity lvl, const FormatOrString & fs) override
@ -184,7 +184,7 @@ struct JSONLogger : Logger {
json["action"] = "msg";
json["level"] = ei.level;
json["msg"] = oss.str();
json["raw_msg"] = ei.hint->str();
json["raw_msg"] = ei.msg.str();
if (ei.errPos.has_value() && (*ei.errPos)) {
json["line"] = ei.errPos->line;
@ -305,10 +305,7 @@ bool handleJSONLogMessage(const std::string & msg,
}
} catch (std::exception & e) {
logError({
.name = "JSON log message",
.hint = hintfmt("bad log message from builder: %s", e.what())
});
printError("bad JSON log message from builder: %s", e.what());
}
return true;

55
src/libutil/ref.hh
View file

@ -17,7 +17,7 @@ private:
public:
ref<T>(const ref<T> & r)
ref(const ref<T> & r)
: p(r.p)
{ }
@ -73,6 +73,16 @@ public:
return ref<T2>((std::shared_ptr<T2>) p);
}
bool operator == (const ref<T> & other) const
{
return p == other.p;
}
bool operator != (const ref<T> & other) const
{
return p != other.p;
}
private:
template<typename T2, typename... Args>
@ -89,4 +99,47 @@ make_ref(Args&&... args)
return ref<T>(p);
}
/* A non-nullable pointer.
This is similar to a C++ "& reference", but mutable.
This is similar to ref<T> but backed by a regular pointer instead of a smart pointer.
*/
template<typename T>
class ptr {
private:
T * p;
public:
ptr<T>(const ptr<T> & r)
: p(r.p)
{ }
explicit ptr<T>(T * p)
: p(p)
{
if (!p)
throw std::invalid_argument("null pointer cast to ptr");
}
T* operator ->() const
{
return &*p;
}
T& operator *() const
{
return *p;
}
bool operator == (const ptr<T> & other) const
{
return p == other.p;
}
bool operator != (const ptr<T> & other) const
{
return p != other.p;
}
};
}

79
src/libutil/serialise.cc
View file

@ -52,10 +52,7 @@ size_t threshold = 256 * 1024 * 1024;
static void warnLargeDump()
{
logWarning({
.name = "Large path",
.description = "dumping very large path (> 256 MiB); this may run out of memory"
});
warn("dumping very large path (> 256 MiB); this may run out of memory");
}
@ -204,6 +201,62 @@ static DefaultStackAllocator defaultAllocatorSingleton;
StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
{
struct SourceToSink : FinishSink
{
typedef boost::coroutines2::coroutine<bool> coro_t;
std::function<void(Source &)> fun;
std::optional<coro_t::push_type> coro;
SourceToSink(std::function<void(Source &)> fun) : fun(fun)
{
}
std::string_view cur;
void operator () (std::string_view in) override
{
if (in.empty()) return;
cur = in;
if (!coro)
coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) {
LambdaSource source([&](char *out, size_t out_len) {
if (cur.empty()) {
yield();
if (yield.get()) {
return (size_t)0;
}
}
size_t n = std::min(cur.size(), out_len);
memcpy(out, cur.data(), n);
cur.remove_prefix(n);
return n;
});
fun(source);
});
if (!*coro) { abort(); }
if (!cur.empty()) (*coro)(false);
}
void finish() override
{
if (!coro) return;
if (!*coro) abort();
(*coro)(true);
if (*coro) abort();
}
};
return std::make_unique<SourceToSink>(fun);
}
std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun,
std::function<void()> eof)
@ -215,7 +268,6 @@ std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun;
std::function<void()> eof;
std::optional<coro_t::pull_type> coro;
bool started = false;
SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
: fun(fun), eof(eof)
@ -306,8 +358,7 @@ Sink & operator << (Sink & sink, const Error & ex)
<< "Error"
<< info.level
<< info.name
<< info.description
<< (info.hint ? info.hint->str() : "")
<< info.msg.str()
<< 0 // FIXME: info.errPos
<< info.traces.size();
for (auto & trace : info.traces) {
@ -374,12 +425,14 @@ Error readError(Source & source)
{
auto type = readString(source);
assert(type == "Error");
ErrorInfo info;
info.level = (Verbosity) readInt(source);
info.name = readString(source);
info.description = readString(source);
auto hint = readString(source);
if (hint != "") info.hint = hintformat(std::move(format("%s") % hint));
auto level = (Verbosity) readInt(source);
auto name = readString(source);
auto msg = readString(source);
ErrorInfo info {
.level = level,
.name = name,
.msg = hintformat(std::move(format("%s") % msg)),
};
auto havePos = readNum<size_t>(source);
assert(havePos == 0);
auto nrTraces = readNum<size_t>(source);

8
src/libutil/serialise.hh
View file

@ -25,6 +25,13 @@ struct NullSink : Sink
{ }
};
struct FinishSink : virtual Sink
{
virtual void finish() = 0;
};
/* A buffered abstract sink. Warning: a BufferedSink should not be
used from multiple threads concurrently. */
struct BufferedSink : virtual Sink
@ -281,6 +288,7 @@ struct ChainSource : Source
size_t read(char * data, size_t len) override;
};
std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun);
/* Convert a function that feeds data into a Sink into a Source. The
Source executes the function as a coroutine. */

130
src/libutil/tarfile.cc
View file

@ -2,83 +2,76 @@
#include <archive_entry.h>
#include "serialise.hh"
#include "tarfile.hh"
namespace nix {
struct TarArchive {
struct archive * archive;
Source * source;
std::vector<unsigned char> buffer;
static int callback_open(struct archive *, void * self)
{
return ARCHIVE_OK;
}
void check(int err, const char * reason = "failed to extract archive: %s")
{
if (err == ARCHIVE_EOF)
throw EndOfFile("reached end of archive");
else if (err != ARCHIVE_OK)
throw Error(reason, archive_error_string(this->archive));
static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
{
auto self = (TarArchive *) _self;
*buffer = self->buffer.data();
try {
return self->source->read((char *) self->buffer.data(), 4096);
} catch (EndOfFile &) {
return 0;
} catch (std::exception & err) {
archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
return -1;
}
}
TarArchive(Source & source) : buffer(4096)
{
this->archive = archive_read_new();
this->source = &source;
static int callback_close(struct archive *, void * self)
{
return ARCHIVE_OK;
}
archive_read_support_filter_all(archive);
void TarArchive::check(int err, const std::string & reason)
{
if (err == ARCHIVE_EOF)
throw EndOfFile("reached end of archive");
else if (err != ARCHIVE_OK)
throw Error(reason, archive_error_string(this->archive));
}
TarArchive::TarArchive(Source & source, bool raw)
: source(&source), buffer(4096)
{
init();
if (!raw)
archive_read_support_format_all(archive);
check(archive_read_open(archive,
(void *)this,
TarArchive::callback_open,
TarArchive::callback_read,
TarArchive::callback_close),
"failed to open archive: %s");
}
else
archive_read_support_format_raw(archive);
check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
}
TarArchive(const Path & path)
{
this->archive = archive_read_new();
TarArchive::TarArchive(const Path & path)
{
init();
archive_read_support_format_all(archive);
check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
}
archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
}
void TarArchive::init()
{
archive = archive_read_new();
archive_read_support_filter_all(archive);
}
TarArchive(const TarArchive &) = delete;
void TarArchive::close()
{
check(archive_read_close(this->archive), "Failed to close archive (%s)");
}
void close()
{
check(archive_read_close(archive), "failed to close archive: %s");
}
~TarArchive()
{
if (this->archive) archive_read_free(this->archive);
}
private:
static int callback_open(struct archive *, void * self) {
return ARCHIVE_OK;
}
static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
{
auto self = (TarArchive *)_self;
*buffer = self->buffer.data();
try {
return self->source->read((char *) self->buffer.data(), 4096);
} catch (EndOfFile &) {
return 0;
} catch (std::exception & err) {
archive_set_error(archive, EIO, "source threw exception: %s", err.what());
return -1;
}
}
static int callback_close(struct archive *, void * self) {
return ARCHIVE_OK;
}
};
TarArchive::~TarArchive()
{
if (this->archive) archive_read_free(this->archive);
}
static void extract_archive(TarArchive & archive, const Path & destDir)
{
@ -92,13 +85,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
struct archive_entry * entry;
int r = archive_read_next_header(archive.archive, &entry);
if (r == ARCHIVE_EOF) break;
else if (r == ARCHIVE_WARN)
auto name = archive_entry_pathname(entry);
if (!name)
throw Error("cannot get archive member name: %s", archive_error_string(archive.archive));
if (r == ARCHIVE_WARN)
warn(archive_error_string(archive.archive));
else
archive.check(r);
archive_entry_set_pathname(entry,
(destDir + "/" + archive_entry_pathname(entry)).c_str());
(destDir + "/" + name).c_str());
archive.check(archive_read_extract(archive.archive, entry, flags));
}

22
src/libutil/tarfile.hh
View file

@ -1,7 +1,29 @@
#include "serialise.hh"
#include <archive.h>
namespace nix {
struct TarArchive {
struct archive * archive;
Source * source;
std::vector<unsigned char> buffer;
void check(int err, const std::string & reason = "failed to extract archive (%s)");
TarArchive(Source & source, bool raw = false);
TarArchive(const Path & path);
// disable copy constructor
TarArchive(const TarArchive &) = delete;
void init();
void close();
~TarArchive();
};
void unpackTarfile(Source & source, const Path & destDir);
void unpackTarfile(const Path & tarFile, const Path & destDir);

70
src/libutil/tests/closure.cc Normal file
View file

@ -0,0 +1,70 @@
#include "closure.hh"
#include <gtest/gtest.h>
namespace nix {
using namespace std;
map<string, set<string>> testGraph = {
{ "A", { "B", "C", "G" } },
{ "B", { "A" } }, // Loops back to A
{ "C", { "F" } }, // Indirect reference
{ "D", { "A" } }, // Not reachable, but has backreferences
{ "E", {} }, // Just not reachable
{ "F", {} },
{ "G", { "G" } }, // Self reference
};
TEST(closure, correctClosure) {
set<string> aClosure;
set<string> expectedClosure = {"A", "B", "C", "F", "G"};
computeClosure<string>(
{"A"},
aClosure,
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
promise<set<string>> promisedNodes;
promisedNodes.set_value(testGraph[currentNode]);
processEdges(promisedNodes);
}
);
ASSERT_EQ(aClosure, expectedClosure);
}
TEST(closure, properlyHandlesDirectExceptions) {
struct TestExn {};
set<string> aClosure;
EXPECT_THROW(
computeClosure<string>(
{"A"},
aClosure,
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
throw TestExn();
}
),
TestExn
);
}
TEST(closure, properlyHandlesExceptionsInPromise) {
struct TestExn {};
set<string> aClosure;
EXPECT_THROW(
computeClosure<string>(
{"A"},
aClosure,
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
promise<set<string>> promise;
try {
throw TestExn();
} catch (...) {
promise.set_exception(std::current_exception());
}
processEdges(promise);
}
),
TestExn
);
}
}

18
src/libutil/tests/compression.cc
View file

@ -17,6 +17,24 @@ namespace nix {
ASSERT_EQ(*o, "this-is-a-test");
}
TEST(decompress, decompressNoneCompressed) {
auto method = "none";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, str);
ASSERT_EQ(*o, str);
}
TEST(decompress, decompressEmptyCompressed) {
// Empty-method decompression used e.g. by S3 store
// (Content-Encoding == "").
auto method = "";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, str);
ASSERT_EQ(*o, str);
}
TEST(decompress, decompressXzCompressed) {
auto method = "xz";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";

32
src/libutil/tests/config.cc
View file

@ -29,20 +29,20 @@ namespace nix {
std::map<std::string, Config::SettingInfo> settings;
Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
config.getSettings(settings, /* overridenOnly = */ false);
config.getSettings(settings, /* overriddenOnly = */ false);
const auto iter = settings.find("name-of-the-setting");
ASSERT_NE(iter, settings.end());
ASSERT_EQ(iter->second.value, "");
ASSERT_EQ(iter->second.description, "description\n");
}
TEST(Config, getDefinedOverridenSettingNotSet) {
TEST(Config, getDefinedOverriddenSettingNotSet) {
Config config;
std::string value;
std::map<std::string, Config::SettingInfo> settings;
Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
config.getSettings(settings, /* overridenOnly = */ true);
config.getSettings(settings, /* overriddenOnly = */ true);
const auto e = settings.find("name-of-the-setting");
ASSERT_EQ(e, settings.end());
}
@ -55,7 +55,7 @@ namespace nix {
setting.assign("value");
config.getSettings(settings, /* overridenOnly = */ false);
config.getSettings(settings, /* overriddenOnly = */ false);
const auto iter = settings.find("name-of-the-setting");
ASSERT_NE(iter, settings.end());
ASSERT_EQ(iter->second.value, "value");
@ -69,7 +69,7 @@ namespace nix {
ASSERT_TRUE(config.set("name-of-the-setting", "value"));
config.getSettings(settings, /* overridenOnly = */ false);
config.getSettings(settings, /* overriddenOnly = */ false);
const auto e = settings.find("name-of-the-setting");
ASSERT_NE(e, settings.end());
ASSERT_EQ(e->second.value, "value");
@ -100,7 +100,7 @@ namespace nix {
{
std::map<std::string, Config::SettingInfo> settings;
config.getSettings(settings, /* overridenOnly = */ false);
config.getSettings(settings, /* overriddenOnly = */ false);
ASSERT_EQ(settings.find("key"), settings.end());
}
@ -108,17 +108,17 @@ namespace nix {
{
std::map<std::string, Config::SettingInfo> settings;
config.getSettings(settings, /* overridenOnly = */ false);
config.getSettings(settings, /* overriddenOnly = */ false);
ASSERT_EQ(settings["key"].value, "value");
}
}
TEST(Config, resetOverriden) {
TEST(Config, resetOverridden) {
Config config;
config.resetOverriden();
config.resetOverridden();
}
TEST(Config, resetOverridenWithSetting) {
TEST(Config, resetOverriddenWithSetting) {
Config config;
Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
@ -127,7 +127,7 @@ namespace nix {
setting.set("foo");
ASSERT_EQ(setting.get(), "foo");
config.getSettings(settings, /* overridenOnly = */ true);
config.getSettings(settings, /* overriddenOnly = */ true);
ASSERT_TRUE(settings.empty());
}
@ -135,18 +135,18 @@ namespace nix {
std::map<std::string, Config::SettingInfo> settings;
setting.override("bar");
ASSERT_TRUE(setting.overriden);
ASSERT_TRUE(setting.overridden);
ASSERT_EQ(setting.get(), "bar");
config.getSettings(settings, /* overridenOnly = */ true);
config.getSettings(settings, /* overriddenOnly = */ true);
ASSERT_FALSE(settings.empty());
}
{
std::map<std::string, Config::SettingInfo> settings;
config.resetOverriden();
ASSERT_FALSE(setting.overriden);
config.getSettings(settings, /* overridenOnly = */ true);
config.resetOverridden();
ASSERT_FALSE(setting.overridden);
config.getSettings(settings, /* overriddenOnly = */ true);
ASSERT_TRUE(settings.empty());
}
}

39
src/libutil/tests/logging.cc
View file

@ -1,3 +1,5 @@
#if 0
#include "logging.hh"
#include "nixexpr.hh"
#include "util.hh"
@ -41,8 +43,7 @@ namespace nix {
makeJSONLogger(*logger)->logEI({
.name = "error name",
.description = "error without any code lines.",
.hint = hintfmt("this hint has %1% templated %2%!!",
.msg = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
.errPos = Pos(foFile, problem_file, 02, 13)
@ -62,7 +63,7 @@ namespace nix {
throw TestError(e.info());
} catch (Error &e) {
ErrorInfo ei = e.info();
ei.hint = hintfmt("%s; subsequent error message.", normaltxt(e.info().hint ? e.info().hint->str() : ""));
ei.msg = hintfmt("%s; subsequent error message.", normaltxt(e.info().msg.str()));
testing::internal::CaptureStderr();
logger->logEI(ei);
@ -95,7 +96,6 @@ namespace nix {
logger->logEI({ .level = lvlInfo,
.name = "Info name",
.description = "Info description",
});
auto str = testing::internal::GetCapturedStderr();
@ -109,7 +109,6 @@ namespace nix {
logger->logEI({ .level = lvlTalkative,
.name = "Talkative name",
.description = "Talkative description",
});
auto str = testing::internal::GetCapturedStderr();
@ -123,7 +122,6 @@ namespace nix {
logger->logEI({ .level = lvlChatty,
.name = "Chatty name",
.description = "Talkative description",
});
auto str = testing::internal::GetCapturedStderr();
@ -137,7 +135,6 @@ namespace nix {
logger->logEI({ .level = lvlDebug,
.name = "Debug name",
.description = "Debug description",
});
auto str = testing::internal::GetCapturedStderr();
@ -151,7 +148,6 @@ namespace nix {
logger->logEI({ .level = lvlVomit,
.name = "Vomit name",
.description = "Vomit description",
});
auto str = testing::internal::GetCapturedStderr();
@ -167,7 +163,6 @@ namespace nix {
logError({
.name = "name",
.description = "error description",
});
auto str = testing::internal::GetCapturedStderr();
@ -182,8 +177,7 @@ namespace nix {
logError({
.name = "error name",
.description = "error with code lines",
.hint = hintfmt("this hint has %1% templated %2%!!",
.msg = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
.errPos = Pos(foString, problem_file, 02, 13),
@ -200,8 +194,7 @@ namespace nix {
logError({
.name = "error name",
.description = "error without any code lines.",
.hint = hintfmt("this hint has %1% templated %2%!!",
.msg = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
.errPos = Pos(foFile, problem_file, 02, 13)
@ -216,7 +209,7 @@ namespace nix {
logError({
.name = "error name",
.hint = hintfmt("hint %1%", "only"),
.msg = hintfmt("hint %1%", "only"),
});
auto str = testing::internal::GetCapturedStderr();
@ -233,8 +226,7 @@ namespace nix {
logWarning({
.name = "name",
.description = "warning description",
.hint = hintfmt("there was a %1%", "warning"),
.msg = hintfmt("there was a %1%", "warning"),
});
auto str = testing::internal::GetCapturedStderr();
@ -250,8 +242,7 @@ namespace nix {
logWarning({
.name = "warning name",
.description = "warning description",
.hint = hintfmt("this hint has %1% templated %2%!!",
.msg = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
.errPos = Pos(foStdin, problem_file, 2, 13),
@ -274,8 +265,7 @@ namespace nix {
auto e = AssertionError(ErrorInfo {
.name = "wat",
.description = "show-traces",
.hint = hintfmt("it has been %1% days since our last error", "zero"),
.msg = hintfmt("it has been %1% days since our last error", "zero"),
.errPos = Pos(foString, problem_file, 2, 13),
});
@ -301,8 +291,7 @@ namespace nix {
auto e = AssertionError(ErrorInfo {
.name = "wat",
.description = "hide traces",
.hint = hintfmt("it has been %1% days since our last error", "zero"),
.msg = hintfmt("it has been %1% days since our last error", "zero"),
.errPos = Pos(foString, problem_file, 2, 13),
});
@ -347,7 +336,7 @@ namespace nix {
ASSERT_STREQ(
hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(),
"only one arg " ANSI_YELLOW "fulfilled" ANSI_NORMAL " ");
"only one arg " ANSI_WARNING "fulfilled" ANSI_NORMAL " ");
}
@ -355,7 +344,7 @@ namespace nix {
ASSERT_STREQ(
hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(),
"what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
"what about this " ANSI_WARNING "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
}
@ -377,3 +366,5 @@ namespace nix {
}
}
#endif

43
src/libutil/tests/tests.cc
View file

@ -4,6 +4,8 @@
#include <limits.h>
#include <gtest/gtest.h>
#include <numeric>
namespace nix {
/* ----------- tests for util.hh ------------------------------------------------*/
@ -282,6 +284,17 @@ namespace nix {
ASSERT_EQ(decoded, s);
}
TEST(base64Encode, encodeAndDecodeNonPrintable) {
char s[256];
std::iota(std::rbegin(s), std::rend(s), 0);
auto encoded = base64Encode(s);
auto decoded = base64Decode(encoded);
EXPECT_EQ(decoded.length(), 255);
ASSERT_EQ(decoded, s);
}
/* ----------------------------------------------------------------------------
* base64Decode
* --------------------------------------------------------------------------*/
@ -294,6 +307,10 @@ namespace nix {
ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum");
}
TEST(base64Decode, decodeThrowsOnInvalidChar) {
ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error);
}
/* ----------------------------------------------------------------------------
* toLower
* --------------------------------------------------------------------------*/
@ -320,20 +337,15 @@ namespace nix {
* --------------------------------------------------------------------------*/
TEST(string2Float, emptyString) {
double n;
ASSERT_EQ(string2Float("", n), false);
ASSERT_EQ(string2Float<double>(""), std::nullopt);
}
TEST(string2Float, trivialConversions) {
double n;
ASSERT_EQ(string2Float("1.0", n), true);
ASSERT_EQ(n, 1.0);
ASSERT_EQ(string2Float<double>("1.0"), 1.0);
ASSERT_EQ(string2Float("0.0", n), true);
ASSERT_EQ(n, 0.0);
ASSERT_EQ(string2Float<double>("0.0"), 0.0);
ASSERT_EQ(string2Float("-100.25", n), true);
ASSERT_EQ(n, (-100.25));
ASSERT_EQ(string2Float<double>("-100.25"), -100.25);
}
/* ----------------------------------------------------------------------------
@ -341,20 +353,15 @@ namespace nix {
* --------------------------------------------------------------------------*/
TEST(string2Int, emptyString) {
double n;
ASSERT_EQ(string2Int("", n), false);
ASSERT_EQ(string2Int<int>(""), std::nullopt);
}
TEST(string2Int, trivialConversions) {
double n;
ASSERT_EQ(string2Int("1", n), true);
ASSERT_EQ(n, 1);
ASSERT_EQ(string2Int<int>("1"), 1);
ASSERT_EQ(string2Int("0", n), true);
ASSERT_EQ(n, 0);
ASSERT_EQ(string2Int<int>("0"), 0);
ASSERT_EQ(string2Int("-100", n), true);
ASSERT_EQ(n, (-100));
ASSERT_EQ(string2Int<int>("-100"), -100);
}
/* ----------------------------------------------------------------------------

18
src/libutil/tests/url.cc
View file

@ -117,6 +117,24 @@ namespace nix {
ASSERT_EQ(parsed, expected);
}
TEST(parseURL, parseScopedRFC4007IPv6Address) {
auto s = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080";
auto parsed = parseURL(s);
ParsedURL expected {
.url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
.base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
.scheme = "http",
.authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
.path = "",
.query = (StringMap) { },
.fragment = "",
};
ASSERT_EQ(parsed, expected);
}
TEST(parseURL, parseIPv6Address) {
auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080";
auto parsed = parseURL(s);

4
src/libutil/url-parts.hh
View file

@ -8,7 +8,7 @@ namespace nix {
// URI stuff.
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)";
const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+";
const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?";
const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")";
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
@ -23,7 +23,7 @@ const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
// A Git ref (i.e. branch or tag name).
const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check
extern std::regex refRegex;
// Instead of defining what a good Git Ref is, we define what a bad Git Ref is

2
src/libutil/url.cc
View file

@ -32,7 +32,7 @@ ParsedURL parseURL(const std::string & url)
auto isFile = scheme.find("file") != std::string::npos;
if (authority && *authority != "" && isFile)
throw Error("file:// URL '%s' has unexpected authority '%s'",
throw BadURL("file:// URL '%s' has unexpected authority '%s'",
url, *authority);
if (isFile && path.empty())

224
src/libutil/util.cc
View file

@ -4,16 +4,18 @@
#include "finally.hh"
#include "serialise.hh"
#include <array>
#include <cctype>
#include <cerrno>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <climits>
#include <future>
#include <iostream>
#include <mutex>
#include <sstream>
#include <thread>
#include <future>
#include <fcntl.h>
#include <grp.h>
@ -32,6 +34,7 @@
#ifdef __linux__
#include <sys/prctl.h>
#include <sys/resource.h>
#endif
@ -143,16 +146,21 @@ Path canonPath(const Path & path, bool resolveSymlinks)
s += '/';
while (i != end && *i != '/') s += *i++;
/* If s points to a symlink, resolve it and restart (since
the symlink target might contain new symlinks). */
/* If s points to a symlink, resolve it and continue from there */
if (resolveSymlinks && isLink(s)) {
if (++followCount >= maxFollow)
throw Error("infinite symlink recursion in path '%1%'", path);
temp = absPath(readLink(s), dirOf(s))
+ string(i, end);
i = temp.begin(); /* restart */
temp = readLink(s) + string(i, end);
i = temp.begin();
end = temp.end();
s = "";
if (!temp.empty() && temp[0] == '/') {
s.clear(); /* restart for symlinks pointing to absolute path */
} else {
s = dirOf(s);
if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = /
s.clear();
}
}
}
}
}
@ -407,7 +415,7 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
}
int fd = openat(parentfd, path.c_str(), O_RDONLY);
if (!fd)
if (fd == -1)
throw SysError("opening directory '%1%'", path);
AutoCloseDir dir(fdopendir(fd));
if (!dir)
@ -429,12 +437,9 @@ static void _deletePath(const Path & path, uint64_t & bytesFreed)
if (dir == "")
dir = "/";
AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY));
AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)};
if (!dirfd) {
// This really shouldn't fail silently, but it's left this way
// for backwards compatibility.
if (errno == ENOENT) return;
throw SysError("opening directory '%1%'", path);
}
@ -752,13 +757,13 @@ AutoCloseFD::AutoCloseFD() : fd{-1} {}
AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd}
AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
{
that.fd = -1;
}
AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that)
AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
{
close();
fd = that.fd;
@ -789,6 +794,7 @@ void AutoCloseFD::close()
if (::close(fd) == -1)
/* This should never happen. */
throw SysError("closing file descriptor %1%", fd);
fd = -1;
}
}
@ -822,6 +828,12 @@ void Pipe::create()
}
void Pipe::close()
{
readSide.close();
writeSide.close();
}
//////////////////////////////////////////////////////////////////////
@ -891,7 +903,7 @@ int Pid::wait()
return status;
}
if (errno != EINTR)
throw SysError("cannot get child exit status");
throw SysError("cannot get exit status of PID %d", pid);
checkInterrupt();
}
}
@ -927,9 +939,6 @@ void killUser(uid_t uid)
users to which the current process can send signals. So we
fork a process, switch to uid, and send a mass kill. */
ProcessOptions options;
options.allowVfork = false;
Pid pid = startProcess([&]() {
if (setuid(uid) == -1)
@ -946,13 +955,13 @@ void killUser(uid_t uid)
#else
if (kill(-1, SIGKILL) == 0) break;
#endif
if (errno == ESRCH) break; /* no more processes */
if (errno == ESRCH || errno == EPERM) break; /* no more processes */
if (errno != EINTR)
throw SysError("cannot kill processes for uid '%1%'", uid);
}
_exit(0);
}, options);
});
int status = pid.wait();
if (status != 0)
@ -1022,17 +1031,10 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
return res;
}
// Output = "standard out" output stream
string runProgram(Path program, bool searchPath, const Strings & args,
const std::optional<std::string> & input)
{
RunOptions opts(program, args);
opts.searchPath = searchPath;
// This allows you to refer to a program with a pathname relative to the
// PATH variable.
opts.input = input;
auto res = runProgram(opts);
auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
if (!statusOk(res.first))
throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first)));
@ -1041,9 +1043,8 @@ string runProgram(Path program, bool searchPath, const Strings & args,
}
// Output = error code + "standard out" output stream
std::pair<int, std::string> runProgram(const RunOptions & options_)
std::pair<int, std::string> runProgram(RunOptions && options)
{
RunOptions options(options_);
StringSink sink;
options.standardOut = &sink;
@ -1081,8 +1082,7 @@ void runProgram2(const RunOptions & options)
// vfork implies that the environment of the main process and the fork will
// be shared (technically this is undefined, but in practice that's the
// case), so we can't use it if we alter the environment
if (options.environment)
processOptions.allowVfork = false;
processOptions.allowVfork = !options.environment;
/* Fork. */
Pid pid = startProcess([&]() {
@ -1109,7 +1109,7 @@ void runProgram2(const RunOptions & options)
Strings args_(options.args);
args_.push_front(options.program);
restoreSignals();
restoreProcessContext();
if (options.searchPath)
execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
@ -1121,7 +1121,7 @@ void runProgram2(const RunOptions & options)
throw SysError("executing '%1%'", options.program);
}, processOptions);
out.writeSide = -1;
out.writeSide.close();
std::thread writerThread;
@ -1134,7 +1134,7 @@ void runProgram2(const RunOptions & options)
if (source) {
in.readSide = -1;
in.readSide.close();
writerThread = std::thread([&]() {
try {
std::vector<char> buf(8 * 1024);
@ -1151,7 +1151,7 @@ void runProgram2(const RunOptions & options)
} catch (...) {
promise.set_exception(std::current_exception());
}
in.writeSide = -1;
in.writeSide.close();
});
}
@ -1249,7 +1249,7 @@ template StringSet tokenizeString(std::string_view s, const string & separators)
template vector<string> tokenizeString(std::string_view s, const string & separators);
string chomp(const string & s)
string chomp(std::string_view s)
{
size_t i = s.find_last_not_of(" \n\r\t");
return i == string::npos ? "" : string(s, 0, i + 1);
@ -1359,6 +1359,12 @@ void ignoreException()
}
}
bool shouldANSI()
{
return isatty(STDERR_FILENO)
&& getEnv("TERM").value_or("dumb") != "dumb"
&& !getEnv("NO_COLOR").has_value();
}
std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width)
{
@ -1430,8 +1436,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in
}
static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
string base64Encode(std::string_view s)
{
@ -1456,15 +1461,15 @@ string base64Encode(std::string_view s)
string base64Decode(std::string_view s)
{
bool init = false;
char decode[256];
if (!init) {
// FIXME: not thread-safe.
memset(decode, -1, sizeof(decode));
constexpr char npos = -1;
constexpr std::array<char, 256> base64DecodeChars = [&]() {
std::array<char, 256> result{};
for (auto& c : result)
c = npos;
for (int i = 0; i < 64; i++)
decode[(int) base64Chars[i]] = i;
init = true;
}
result[base64Chars[i]] = i;
return result;
}();
string res;
unsigned int d = 0, bits = 0;
@ -1473,8 +1478,8 @@ string base64Decode(std::string_view s)
if (c == '=') break;
if (c == '\n') continue;
char digit = decode[(unsigned char) c];
if (digit == -1)
char digit = base64DecodeChars[(unsigned char) c];
if (digit == npos)
throw Error("invalid character in Base64 string: '%c'", c);
bits += 6;
@ -1590,7 +1595,7 @@ void startSignalHandlerThread()
updateWindowSize();
if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
throw SysError("quering signal mask");
throw SysError("querying signal mask");
sigset_t set;
sigemptyset(&set);
@ -1605,12 +1610,45 @@ void startSignalHandlerThread()
std::thread(signalHandlerThread, set).detach();
}
void restoreSignals()
static void restoreSignals()
{
if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
throw SysError("restoring signals");
}
#if __linux__
rlim_t savedStackSize = 0;
#endif
void setStackSize(size_t stackSize)
{
#if __linux__
struct rlimit limit;
if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
savedStackSize = limit.rlim_cur;
limit.rlim_cur = stackSize;
setrlimit(RLIMIT_STACK, &limit);
}
#endif
}
void restoreProcessContext()
{
restoreSignals();
restoreAffinity();
#if __linux__
if (savedStackSize) {
struct rlimit limit;
if (getrlimit(RLIMIT_STACK, &limit) == 0) {
limit.rlim_cur = savedStackSize;
setrlimit(RLIMIT_STACK, &limit);
}
}
#endif
}
/* RAII helper to automatically deregister a callback. */
struct InterruptCallbackImpl : InterruptCallback
{
@ -1634,7 +1672,7 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
}
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
AutoCloseFD createUnixDomainSocket()
{
AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM
#ifdef SOCK_CLOEXEC
@ -1643,19 +1681,16 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
, 0);
if (!fdSocket)
throw SysError("cannot create Unix domain socket");
closeOnExec(fdSocket.get());
return fdSocket;
}
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%1%' is too long", path);
strcpy(addr.sun_path, path.c_str());
unlink(path.c_str());
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
{
auto fdSocket = nix::createUnixDomainSocket();
if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%1%'", path);
bind(fdSocket.get(), path);
if (chmod(path.c_str(), mode) == -1)
throw SysError("changing permissions on '%1%'", path);
@ -1667,16 +1702,79 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
}
void bind(int fd, const std::string & path)
{
unlink(path.c_str());
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
auto dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot bind to socket '%s'", path);
} else {
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
}
}
void connect(int fd, const std::string & path)
{
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
auto dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot connect to socket at '%s'", path);
} else {
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
}
}
string showBytes(uint64_t bytes)
{
return fmt("%.2f MiB", bytes / (1024.0 * 1024.0));
}
// FIXME: move to libstore/build
void commonChildInit(Pipe & logPipe)
{
logger = makeSimpleLogger();
const static string pathNullDevice = "/dev/null";
restoreSignals();
restoreProcessContext();
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning

113
src/libutil/util.hh
View file

@ -188,7 +188,6 @@ public:
class AutoCloseFD
{
int fd;
void close();
public:
AutoCloseFD();
AutoCloseFD(int fd);
@ -200,6 +199,7 @@ public:
int get() const;
explicit operator bool() const;
int release();
void close();
};
@ -216,6 +216,7 @@ class Pipe
public:
AutoCloseFD readSide, writeSide;
void create();
void close();
};
@ -258,10 +259,10 @@ void killUser(uid_t uid);
pid to the caller. */
struct ProcessOptions
{
string errorPrefix = "error: ";
string errorPrefix = "";
bool dieWithParent = true;
bool runExitHandlers = false;
bool allowVfork = true;
bool allowVfork = false;
};
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
@ -275,30 +276,33 @@ string runProgram(Path program, bool searchPath = false,
struct RunOptions
{
Path program;
bool searchPath = true;
Strings args;
std::optional<uid_t> uid;
std::optional<uid_t> gid;
std::optional<Path> chdir;
std::optional<std::map<std::string, std::string>> environment;
Path program;
bool searchPath = true;
Strings args;
std::optional<std::string> input;
Source * standardIn = nullptr;
Sink * standardOut = nullptr;
bool mergeStderrToStdout = false;
bool _killStderr = false;
RunOptions(const Path & program, const Strings & args)
: program(program), args(args) { };
RunOptions & killStderr(bool v) { _killStderr = true; return *this; }
};
std::pair<int, std::string> runProgram(const RunOptions & options);
std::pair<int, std::string> runProgram(RunOptions && options);
void runProgram2(const RunOptions & options);
/* Change the stack size. */
void setStackSize(size_t stackSize);
/* Restore the original inherited Unix process context (such as signal
masks, stack size, CPU affinity). */
void restoreProcessContext();
class ExecError : public Error
{
public:
@ -373,8 +377,9 @@ template<class C> Strings quoteStrings(const C & c)
}
/* Remove trailing whitespace from a string. */
string chomp(const string & s);
/* Remove trailing whitespace from a string. FIXME: return
std::string_view. */
string chomp(std::string_view s);
/* Remove whitespace from the start and end of a string. */
@ -397,21 +402,49 @@ bool statusOk(int status);
/* Parse a string into an integer. */
template<class N> bool string2Int(const string & s, N & n)
template<class N>
std::optional<N> string2Int(const std::string & s)
{
if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed)
return false;
if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
return std::nullopt;
std::istringstream str(s);
N n;
str >> n;
return str && str.get() == EOF;
if (str && str.get() == EOF) return n;
return std::nullopt;
}
/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or
'T' denoting a binary unit prefix. */
template<class N>
N string2IntWithUnitPrefix(std::string s)
{
N multiplier = 1;
if (!s.empty()) {
char u = std::toupper(*s.rbegin());
if (std::isalpha(u)) {
if (u == 'K') multiplier = 1ULL << 10;
else if (u == 'M') multiplier = 1ULL << 20;
else if (u == 'G') multiplier = 1ULL << 30;
else if (u == 'T') multiplier = 1ULL << 40;
else throw UsageError("invalid unit specifier '%1%'", u);
s.resize(s.size() - 1);
}
}
if (auto n = string2Int<N>(s))
return *n * multiplier;
throw UsageError("'%s' is not an integer", s);
}
/* Parse a string into a float. */
template<class N> bool string2Float(const string & s, N & n)
template<class N>
std::optional<N> string2Float(const string & s)
{
std::istringstream str(s);
N n;
str >> n;
return str && str.get() == EOF;
if (str && str.get() == EOF) return n;
return std::nullopt;
}
@ -443,6 +476,9 @@ constexpr char treeLast[] = "└───";
constexpr char treeLine[] = "│ ";
constexpr char treeNull[] = " ";
/* Determine whether ANSI escape sequences are appropriate for the
present output. */
bool shouldANSI();
/* Truncate a string to 'width' printable characters. If 'filterAll'
is true, all ANSI escape sequences are filtered out. Otherwise,
@ -475,6 +511,29 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_
}
/* Remove and return the first item from a container. */
template <class T>
std::optional<typename T::value_type> remove_begin(T & c)
{
auto i = c.begin();
if (i == c.end()) return {};
auto v = std::move(*i);
c.erase(i);
return v;
}
/* Remove and return the first item from a container. */
template <class T>
std::optional<typename T::value_type> pop(T & c)
{
if (c.empty()) return {};
auto v = std::move(c.front());
c.pop();
return v;
}
template<typename T>
class Callback;
@ -483,9 +542,6 @@ class Callback;
on the current thread (and thus any threads created by it). */
void startSignalHandlerThread();
/* Restore default signal handling. */
void restoreSignals();
struct InterruptCallback
{
virtual ~InterruptCallback() { };
@ -538,9 +594,18 @@ extern PathFilter defaultPathFilter;
/* Common initialisation performed in child processes. */
void commonChildInit(Pipe & logPipe);
/* Create a Unix domain socket. */
AutoCloseFD createUnixDomainSocket();
/* Create a Unix domain socket in listen mode. */
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
/* Bind a Unix domain socket to a path. */
void bind(int fd, const std::string & path);
/* Connect to a Unix domain socket. */
void connect(int fd, const std::string & path);
// A Rust/Python-like enumerate() iterator adapter.
// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17.