1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-11 21:16:02 +01:00

Merge remote-tracking branch 'origin/master' into coerce-string

This commit is contained in:
Guillaume Maudoux 2022-04-29 00:12:25 +02:00
commit e93b59fbc5
170 changed files with 4010 additions and 2203 deletions

View file

@ -127,11 +127,11 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
if (flag.handler.arity == ArityAny) break;
throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
}
if (flag.completer)
if (auto prefix = needsCompletion(*pos)) {
anyCompleted = true;
if (auto prefix = needsCompletion(*pos)) {
anyCompleted = true;
if (flag.completer)
flag.completer(n, *prefix);
}
}
args.push_back(*pos++);
}
if (!anyCompleted)
@ -146,6 +146,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
&& hasPrefix(name, std::string(*prefix, 2)))
completions->add("--" + name, flag->description);
}
return false;
}
auto i = longFlags.find(std::string(*pos, 2));
if (i == longFlags.end()) return false;
@ -187,10 +188,12 @@ bool Args::processArgs(const Strings & args, bool finish)
{
std::vector<std::string> ss;
for (const auto &[n, s] : enumerate(args)) {
ss.push_back(s);
if (exp.completer)
if (auto prefix = needsCompletion(s))
if (auto prefix = needsCompletion(s)) {
ss.push_back(*prefix);
if (exp.completer)
exp.completer(n, *prefix);
} else
ss.push_back(s);
}
exp.handler.fun(ss);
expectedArgs.pop_front();
@ -279,21 +282,22 @@ static void _completePath(std::string_view prefix, bool onlyDirs)
{
completionType = ctFilenames;
glob_t globbuf;
int flags = GLOB_NOESCAPE | GLOB_TILDE;
int flags = GLOB_NOESCAPE;
#ifdef GLOB_ONLYDIR
if (onlyDirs)
flags |= GLOB_ONLYDIR;
#endif
if (glob((std::string(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) {
// using expandTilde here instead of GLOB_TILDE(_CHECK) so that ~<Tab> expands to /home/user/
if (glob((expandTilde(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) {
for (size_t i = 0; i < globbuf.gl_pathc; ++i) {
if (onlyDirs) {
auto st = lstat(globbuf.gl_pathv[i]);
auto st = stat(globbuf.gl_pathv[i]);
if (!S_ISDIR(st.st_mode)) continue;
}
completions->add(globbuf.gl_pathv[i]);
}
globfree(&globbuf);
}
globfree(&globbuf);
}
void completePath(size_t, std::string_view prefix)
@ -322,11 +326,6 @@ MultiCommand::MultiCommand(const Commands & commands_)
.optional = true,
.handler = {[=](std::string s) {
assert(!command);
if (auto prefix = needsCompletion(s)) {
for (auto & [name, command] : commands)
if (hasPrefix(name, *prefix))
completions->add(name);
}
auto i = commands.find(s);
if (i == commands.end()) {
std::set<std::string> commandNames;
@ -337,6 +336,11 @@ MultiCommand::MultiCommand(const Commands & commands_)
}
command = {s, i->second()};
command->second->parent = this;
}},
.completer = {[&](size_t, std::string_view prefix) {
for (auto & [name, command] : commands)
if (hasPrefix(name, prefix))
completions->add(name);
}}
});

View file

@ -0,0 +1,68 @@
#pragma once
#include <cstdint>
#include <cstdlib>
#include <vector>
#include <limits>
namespace nix {
/* Provides an indexable container like vector<> with memory overhead
guarantees like list<> by allocating storage in chunks of ChunkSize
elements instead of using a contiguous memory allocation like vector<>
does. Not using a single vector that is resized reduces memory overhead
on large data sets by on average (growth factor)/2, mostly
eliminates copies within the vector during resizing, and provides stable
references to its elements. */
template<typename T, size_t ChunkSize>
class ChunkedVector {
private:
uint32_t size_ = 0;
std::vector<std::vector<T>> chunks;
/* keep this out of the ::add hot path */
[[gnu::noinline]]
auto & addChunk()
{
if (size_ >= std::numeric_limits<uint32_t>::max() - ChunkSize)
abort();
chunks.emplace_back();
chunks.back().reserve(ChunkSize);
return chunks.back();
}
public:
ChunkedVector(uint32_t reserve)
{
chunks.reserve(reserve);
addChunk();
}
uint32_t size() const { return size_; }
std::pair<T &, uint32_t> add(T value)
{
const auto idx = size_++;
auto & chunk = [&] () -> auto & {
if (auto & back = chunks.back(); back.size() < ChunkSize)
return back;
return addChunk();
}();
auto & result = chunk.emplace_back(std::move(value));
return {result, idx};
}
const T & operator[](uint32_t idx) const
{
return chunks[idx / ChunkSize][idx % ChunkSize];
}
template<typename Fn>
void forEach(Fn fn) const
{
for (const auto & c : chunks)
for (const auto & e : c)
fn(e);
}
};
}

View file

@ -9,10 +9,9 @@ namespace nix {
const std::string nativeSystem = SYSTEM;
BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
void BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
{
err.traces.push_front(Trace { .pos = e, .hint = hint });
return *this;
}
// c++ std::exception descendants must have a 'const char* what()' function.
@ -22,12 +21,9 @@ const std::string & BaseError::calcWhat() const
if (what_.has_value())
return *what_;
else {
err.name = sname();
std::ostringstream oss;
showErrorInfo(oss, err, loggerSettings.showTrace);
what_ = oss.str();
return *what_;
}
}

View file

@ -87,11 +87,7 @@ struct ErrPos {
origin = pos.origin;
line = pos.line;
column = pos.column;
// is file symbol null?
if (pos.file.set())
file = pos.file;
else
file = "";
file = pos.file;
return *this;
}
@ -109,7 +105,6 @@ struct Trace {
struct ErrorInfo {
Verbosity level;
std::string name; // FIXME: rename
hintformat msg;
std::optional<ErrPos> errPos;
std::list<Trace> traces;
@ -164,8 +159,6 @@ public:
: err(e)
{ }
virtual const char* sname() const { return "BaseError"; }
#ifdef EXCEPTION_NEEDS_THROW_SPEC
~BaseError() throw () { };
const char * what() const throw () { return calcWhat().c_str(); }
@ -181,12 +174,12 @@ public:
}
template<typename... Args>
BaseError & addTrace(std::optional<ErrPos> e, const std::string_view & fs, const Args & ... args)
void addTrace(std::optional<ErrPos> e, std::string_view fs, const Args & ... args)
{
return addTrace(e, hintfmt(std::string(fs), args...));
addTrace(e, hintfmt(std::string(fs), args...));
}
BaseError & addTrace(std::optional<ErrPos> e, hintformat hint);
void addTrace(std::optional<ErrPos> e, hintformat hint);
bool hasTrace() const { return !err.traces.empty(); }
};
@ -196,7 +189,6 @@ public:
{ \
public: \
using superClass::superClass; \
virtual const char* sname() const override { return #newClass; } \
}
MakeError(Error, BaseError);
@ -216,8 +208,6 @@ public:
auto hf = hintfmt(args...);
err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
}
virtual const char* sname() const override { return "SysError"; }
};
}

View file

@ -7,10 +7,12 @@ namespace nix {
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::CaDerivations, "ca-derivations" },
{ Xp::ImpureDerivations, "impure-derivations" },
{ Xp::Flakes, "flakes" },
{ Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" },
{ Xp::NoUrlLiterals, "no-url-literals" },
{ Xp::FetchClosure, "fetch-closure" },
};
const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
@ -56,4 +58,18 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu
return str << showExperimentalFeature(feature);
}
void to_json(nlohmann::json& j, const ExperimentalFeature& feature) {
j = showExperimentalFeature(feature);
}
void from_json(const nlohmann::json& j, ExperimentalFeature& feature) {
const std::string input = j;
const auto parsed = parseExperimentalFeature(input);
if (parsed.has_value())
feature = *parsed;
else
throw Error("Unknown experimental feature '%s' in JSON input", input);
}
}

View file

@ -16,10 +16,12 @@ namespace nix {
enum struct ExperimentalFeature
{
CaDerivations,
ImpureDerivations,
Flakes,
NixCommand,
RecursiveNix,
NoUrlLiterals
NoUrlLiterals,
FetchClosure,
};
/**
@ -47,10 +49,13 @@ public:
ExperimentalFeature missingFeature;
MissingExperimentalFeature(ExperimentalFeature);
virtual const char * sname() const override
{
return "MissingExperimentalFeature";
}
};
/**
* Semi-magic conversion to and from json.
* See the nlohmann/json readme for more details.
*/
void to_json(nlohmann::json&, const ExperimentalFeature&);
void from_json(const nlohmann::json&, ExperimentalFeature&);
}

View file

@ -2,7 +2,6 @@
#include <boost/format.hpp>
#include <string>
#include <regex>
#include "ansicolor.hh"
@ -155,15 +154,4 @@ inline hintformat hintfmt(std::string plain_string)
return hintfmt("%s", normaltxt(plain_string));
}
/* Highlight all the given matches in the given string `s` by wrapping
them between `prefix` and `postfix`.
If some matches overlap, then their union will be wrapped rather
than the individual matches. */
std::string hiliteMatches(
std::string_view s,
std::vector<std::smatch> matches,
std::string_view prefix,
std::string_view postfix);
}

View file

@ -155,7 +155,7 @@ static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_
{
bool isSRI = false;
// Parse the has type before the separater, if there was one.
// Parse the hash type before the separator, if there was one.
std::optional<HashType> optParsedType;
{
auto hashRaw = splitPrefixTo(rest, ':');

View file

@ -93,13 +93,11 @@ public:
std::string gitRev() const
{
assert(type == htSHA1);
return to_string(Base16, false);
}
std::string gitShortRev() const
{
assert(type == htSHA1);
return std::string(to_string(Base16, false), 0, 7);
}

View file

@ -1,6 +1,4 @@
#include "fmt.hh"
#include <regex>
#include "hilite.hh"
namespace nix {

20
src/libutil/hilite.hh Normal file
View file

@ -0,0 +1,20 @@
#pragma once
#include <regex>
#include <vector>
#include <string>
namespace nix {
/* Highlight all the given matches in the given string `s` by wrapping
them between `prefix` and `postfix`.
If some matches overlap, then their union will be wrapped rather
than the individual matches. */
std::string hiliteMatches(
std::string_view s,
std::vector<std::smatch> matches,
std::string_view prefix,
std::string_view postfix);
}

View file

@ -99,47 +99,4 @@ make_ref(Args&&... args)
return ref<T>(p);
}
/* A non-nullable pointer.
This is similar to a C++ "& reference", but mutable.
This is similar to ref<T> but backed by a regular pointer instead of a smart pointer.
*/
template<typename T>
class ptr {
private:
T * p;
public:
ptr<T>(const ptr<T> & r)
: p(r.p)
{ }
explicit ptr<T>(T * p)
: p(p)
{
if (!p)
throw std::invalid_argument("null pointer cast to ptr");
}
T* operator ->() const
{
return &*p;
}
T& operator *() const
{
return *p;
}
bool operator == (const ptr<T> & other) const
{
return p == other.p;
}
bool operator != (const ptr<T> & other) const
{
return p != other.p;
}
};
}

View file

@ -357,7 +357,7 @@ Sink & operator << (Sink & sink, const Error & ex)
sink
<< "Error"
<< info.level
<< info.name
<< "Error" // removed
<< info.msg.str()
<< 0 // FIXME: info.errPos
<< info.traces.size();
@ -426,11 +426,10 @@ Error readError(Source & source)
auto type = readString(source);
assert(type == "Error");
auto level = (Verbosity) readInt(source);
auto name = readString(source);
auto name = readString(source); // removed
auto msg = readString(source);
ErrorInfo info {
.level = level,
.name = name,
.msg = hintformat(std::move(format("%s") % msg)),
};
auto havePos = readNum<size_t>(source);

View file

@ -39,30 +39,32 @@ void TarArchive::check(int err, const std::string & reason)
throw Error(reason, archive_error_string(this->archive));
}
TarArchive::TarArchive(Source & source, bool raw)
: source(&source), buffer(4096)
TarArchive::TarArchive(Source & source, bool raw) : buffer(4096)
{
init();
if (!raw)
this->archive = archive_read_new();
this->source = &source;
if (!raw) {
archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
else
} else {
archive_read_support_filter_all(archive);
archive_read_support_format_raw(archive);
archive_read_support_format_empty(archive);
}
check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
}
TarArchive::TarArchive(const Path & path)
{
init();
this->archive = archive_read_new();
archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
}
void TarArchive::init()
{
archive = archive_read_new();
archive_read_support_filter_all(archive);
}
void TarArchive::close()
{
check(archive_read_close(this->archive), "Failed to close archive (%s)");

View file

@ -17,13 +17,10 @@ struct TarArchive {
// disable copy constructor
TarArchive(const TarArchive &) = delete;
void init();
void close();
~TarArchive();
};
void unpackTarfile(Source & source, const Path & destDir);
void unpackTarfile(const Path & tarFile, const Path & destDir);

View file

@ -0,0 +1,54 @@
#include "chunked-vector.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(ChunkedVector, InitEmpty) {
auto v = ChunkedVector<int, 2>(100);
ASSERT_EQ(v.size(), 0);
}
TEST(ChunkedVector, GrowsCorrectly) {
auto v = ChunkedVector<int, 2>(100);
for (auto i = 1; i < 20; i++) {
v.add(i);
ASSERT_EQ(v.size(), i);
}
}
TEST(ChunkedVector, AddAndGet) {
auto v = ChunkedVector<int, 2>(100);
for (auto i = 1; i < 20; i++) {
auto [i2, idx] = v.add(i);
auto & i3 = v[idx];
ASSERT_EQ(i, i2);
ASSERT_EQ(&i2, &i3);
}
}
TEST(ChunkedVector, ForEach) {
auto v = ChunkedVector<int, 2>(100);
for (auto i = 1; i < 20; i++) {
v.add(i);
}
int count = 0;
v.forEach([&count](int elt) {
count++;
});
ASSERT_EQ(count, v.size());
}
TEST(ChunkedVector, OverflowOK) {
// Similar to the AddAndGet, but intentionnally use a small
// initial ChunkedVector to force it to overflow
auto v = ChunkedVector<int, 2>(2);
for (auto i = 1; i < 20; i++) {
auto [i2, idx] = v.add(i);
auto & i3 = v[idx];
ASSERT_EQ(i, i2);
ASSERT_EQ(&i2, &i3);
}
}
}

View file

@ -1,9 +1,7 @@
#include "fmt.hh"
#include "hilite.hh"
#include <gtest/gtest.h>
#include <regex>
namespace nix {
/* ----------- tests for fmt.hh -------------------------------------------------*/

View file

@ -178,7 +178,7 @@ namespace nix {
}
TEST(parseURL, parseFileURLWithQueryAndFragment) {
auto s = "file:///none/of/your/business";
auto s = "file:///none/of//your/business";
auto parsed = parseURL(s);
ParsedURL expected {
@ -186,7 +186,7 @@ namespace nix {
.base = "",
.scheme = "file",
.authority = "",
.path = "/none/of/your/business",
.path = "/none/of//your/business",
.query = (StringMap) { },
.fragment = "",
};

View file

@ -5,6 +5,7 @@
#include <list>
#include <set>
#include <string>
#include <limits>
#include <map>
#include <variant>
#include <vector>

View file

@ -18,7 +18,7 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod
const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
const static std::string segmentRegex = "(?:" + pcharRegex + "*)";
const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";

View file

@ -71,13 +71,11 @@ void clearEnv()
unsetenv(name.first.c_str());
}
void replaceEnv(std::map<std::string, std::string> newEnv)
void replaceEnv(const std::map<std::string, std::string> & newEnv)
{
clearEnv();
for (auto newEnvVar : newEnv)
{
for (auto & newEnvVar : newEnv)
setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
}
}
@ -200,6 +198,17 @@ std::string_view baseNameOf(std::string_view path)
}
std::string expandTilde(std::string_view path)
{
// TODO: expand ~user ?
auto tilde = path.substr(0, 2);
if (tilde == "~/" || tilde == "~")
return getHome() + std::string(path.substr(1));
else
return std::string(path);
}
bool isInDir(std::string_view path, std::string_view dir)
{
return path.substr(0, 1) == "/"
@ -215,6 +224,15 @@ bool isDirOrInDir(std::string_view path, std::string_view dir)
}
struct stat stat(const Path & path)
{
struct stat st;
if (stat(path.c_str(), &st))
throw SysError("getting status of '%1%'", path);
return st;
}
struct stat lstat(const Path & path)
{
struct stat st;
@ -1064,7 +1082,7 @@ std::string runProgram(Path program, bool searchPath, const Strings & args,
auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
if (!statusOk(res.first))
throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first)));
throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
return res.second;
}
@ -1192,7 +1210,7 @@ void runProgram2(const RunOptions & options)
if (source) promise.get_future().get();
if (status)
throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status)));
throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status));
}
@ -1261,9 +1279,9 @@ template<class C> C tokenizeString(std::string_view s, std::string_view separato
{
C result;
auto pos = s.find_first_not_of(separators, 0);
while (pos != std::string::npos) {
while (pos != std::string_view::npos) {
auto end = s.find_first_of(separators, pos + 1);
if (end == std::string::npos) end = s.size();
if (end == std::string_view::npos) end = s.size();
result.insert(result.end(), std::string(s, pos, end - pos));
pos = s.find_first_not_of(separators, end);
}
@ -1473,6 +1491,7 @@ constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv
std::string base64Encode(std::string_view s)
{
std::string res;
res.reserve((s.size() + 2) / 3 * 4);
int data = 0, nbits = 0;
for (char c : s) {
@ -1504,6 +1523,9 @@ std::string base64Decode(std::string_view s)
}();
std::string res;
// Some sequences are missing the padding consisting of up to two '='.
// vvv
res.reserve((s.size() + 2) / 4 * 3);
unsigned int d = 0, bits = 0;
for (char c : s) {
@ -1690,7 +1712,9 @@ void setStackSize(size_t stackSize)
#endif
}
#if __linux__
static AutoCloseFD fdSavedMountNamespace;
#endif
void saveMountNamespace()
{
@ -1709,8 +1733,13 @@ void restoreMountNamespace()
{
#if __linux__
try {
auto savedCwd = absPath(".");
if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
throw SysError("restoring parent mount namespace");
if (chdir(savedCwd.c_str()) == -1) {
throw SysError("restoring cwd");
}
} catch (Error & e) {
debug(e.msg());
}

View file

@ -68,6 +68,9 @@ Path dirOf(const PathView path);
following the final `/' (trailing slashes are removed). */
std::string_view baseNameOf(std::string_view path);
/* Perform tilde expansion on a path. */
std::string expandTilde(std::string_view path);
/* Check whether 'path' is a descendant of 'dir'. Both paths must be
canonicalized. */
bool isInDir(std::string_view path, std::string_view dir);
@ -77,6 +80,7 @@ bool isInDir(std::string_view path, std::string_view dir);
bool isDirOrInDir(std::string_view path, std::string_view dir);
/* Get status of `path'. */
struct stat stat(const Path & path);
struct stat lstat(const Path & path);
/* Return true iff the given path exists. */