WIP: broken merge but need a git checkpoint

src/libutil/args.cc

@@ -124,7 +124,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         bool anyCompleted = false;
         for (size_t n = 0 ; n < flag.handler.arity; ++n) {
             if (pos == end) {
-                if (flag.handler.arity == ArityAny) break;
+                if (flag.handler.arity == ArityAny || anyCompleted) break;
                 throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
             }
             if (auto prefix = needsCompletion(*pos)) {
@@ -362,6 +362,14 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
         return Args::processArgs(args, finish);
 }
 
+void MultiCommand::completionHook()
+{
+    if (command)
+        return command->second->completionHook();
+    else
+        return Args::completionHook();
+}
+
 nlohmann::json MultiCommand::toJSON()
 {
     auto cmds = nlohmann::json::object();

src/libutil/args.hh

@@ -25,6 +25,8 @@ public:
     /* Return a short one-line description of the command. */
     virtual std::string description() { return ""; }
 
+    virtual bool forceImpureByDefault() { return false; }
+
     /* Return documentation about this command, in Markdown format. */
     virtual std::string doc() { return ""; }
 
@@ -146,6 +148,11 @@ protected:
        argument (if any) have been processed. */
     virtual void initialFlagsProcessed() {}
 
+    /* Called after the command line has been processed if we need to generate
+       completions. Useful for commands that need to know the whole command line
+       in order to know what completions to generate. */
+    virtual void completionHook() { }
+
 public:
 
     void addFlag(Flag && flag);
@@ -221,6 +228,8 @@ public:
 
     bool processArgs(const Strings & args, bool finish) override;
 
+    void completionHook() override;
+
     nlohmann::json toJSON() override;
 };
 

src/libutil/error.hh

@@ -98,6 +98,15 @@ struct ErrPos {
     }
 };
 
+std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos);
+
+void printCodeLines(std::ostream & out,
+    const std::string & prefix,
+    const ErrPos & errPos,
+    const LinesOfCode & loc);
+
+void printAtPos(const ErrPos & pos, std::ostream & out);
+
 struct Trace {
     std::optional<ErrPos> pos;
     hintformat hint;
@@ -201,13 +210,19 @@ public:
     int errNo;
 
     template<typename... Args>
-    SysError(const Args & ... args)
+    SysError(int errNo_, const Args & ... args)
         : Error("")
     {
-        errNo = errno;
+        errNo = errNo_;
         auto hf = hintfmt(args...);
         err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
     }
+
+    template<typename... Args>
+    SysError(const Args & ... args)
+        : SysError(errno, args ...)
+    {
+    }
 };
 
 }

src/libutil/experimental-features.cc

@@ -13,6 +13,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::RecursiveNix, "recursive-nix" },
     { Xp::NoUrlLiterals, "no-url-literals" },
     { Xp::FetchClosure, "fetch-closure" },
+    { Xp::ReplFlake, "repl-flake" },
 };
 
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
@@ -35,7 +36,9 @@ const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::str
 
 std::string_view showExperimentalFeature(const ExperimentalFeature feature)
 {
-    return stringifiedXpFeatures.at(feature);
+    const auto ret = get(stringifiedXpFeatures, feature);
+    assert(ret);
+    return *ret;
 }
 
 std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
@@ -58,11 +61,13 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu
     return str << showExperimentalFeature(feature);
 }
 
-void to_json(nlohmann::json& j, const ExperimentalFeature& feature) {
+void to_json(nlohmann::json & j, const ExperimentalFeature & feature)
+{
     j = showExperimentalFeature(feature);
 }
 
-void from_json(const nlohmann::json& j, ExperimentalFeature& feature) {
+void from_json(const nlohmann::json & j, ExperimentalFeature & feature)
+{
     const std::string input = j;
     const auto parsed = parseExperimentalFeature(input);
 

src/libutil/experimental-features.hh

@@ -22,6 +22,7 @@ enum struct ExperimentalFeature
     RecursiveNix,
     NoUrlLiterals,
     FetchClosure,
+    ReplFlake,
 };
 
 /**
@@ -55,7 +56,7 @@ public:
  * Semi-magic conversion to and from json.
  * See the nlohmann/json readme for more details.
  */
-void to_json(nlohmann::json&, const ExperimentalFeature&);
-void from_json(const nlohmann::json&, ExperimentalFeature&);
+void to_json(nlohmann::json &, const ExperimentalFeature &);
+void from_json(const nlohmann::json &, ExperimentalFeature &);
 
 }

src/libutil/filesystem.cc

@@ -0,0 +1,172 @@
+#include <sys/time.h>
+#include <filesystem>
+
+#include "finally.hh"
+#include "util.hh"
+#include "types.hh"
+
+namespace fs = std::filesystem;
+
+namespace nix {
+
+static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
+    int & counter)
+{
+    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
+    if (includePid)
+        return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
+    else
+        return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str();
+}
+
+Path createTempDir(const Path & tmpRoot, const Path & prefix,
+    bool includePid, bool useGlobalCounter, mode_t mode)
+{
+    static int globalCounter = 0;
+    int localCounter = 0;
+    int & counter(useGlobalCounter ? globalCounter : localCounter);
+
+    while (1) {
+        checkInterrupt();
+        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
+        if (mkdir(tmpDir.c_str(), mode) == 0) {
+#if __FreeBSD__
+            /* Explicitly set the group of the directory.  This is to
+               work around around problems caused by BSD's group
+               ownership semantics (directories inherit the group of
+               the parent).  For instance, the group of /tmp on
+               FreeBSD is "wheel", so all directories created in /tmp
+               will be owned by "wheel"; but if the user is not in
+               "wheel", then "tar" will fail to unpack archives that
+               have the setgid bit set on directories. */
+            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
+                throw SysError("setting group of directory '%1%'", tmpDir);
+#endif
+            return tmpDir;
+        }
+        if (errno != EEXIST)
+            throw SysError("creating directory '%1%'", tmpDir);
+    }
+}
+
+
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
+{
+    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    // Strictly speaking, this is UB, but who cares...
+    // FIXME: use O_TMPFILE.
+    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
+    if (!fd)
+        throw SysError("creating temporary file '%s'", tmpl);
+    closeOnExec(fd.get());
+    return {std::move(fd), tmpl};
+}
+
+void createSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
+{
+    if (symlink(target.c_str(), link.c_str()))
+        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
+    if (mtime) {
+        struct timeval times[2];
+        times[0].tv_sec = *mtime;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = *mtime;
+        times[1].tv_usec = 0;
+        if (lutimes(link.c_str(), times))
+            throw SysError("setting time of symlink '%s'", link);
+    }
+}
+
+void replaceSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
+{
+    for (unsigned int n = 0; true; n++) {
+        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
+
+        try {
+            createSymlink(target, tmp, mtime);
+        } catch (SysError & e) {
+            if (e.errNo == EEXIST) continue;
+            throw;
+        }
+
+        renameFile(tmp, link);
+
+        break;
+    }
+}
+
+void setWriteTime(const fs::path & p, const struct stat & st)
+{
+    struct timeval times[2];
+    times[0] = {
+        .tv_sec = st.st_atime,
+        .tv_usec = 0,
+    };
+    times[1] = {
+        .tv_sec = st.st_mtime,
+        .tv_usec = 0,
+    };
+    if (lutimes(p.c_str(), times) != 0)
+        throw SysError("changing modification time of '%s'", p);
+}
+
+void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
+{
+    // TODO: Rewrite the `is_*` to use `symlink_status()`
+    auto statOfFrom = lstat(from.path().c_str());
+    auto fromStatus = from.symlink_status();
+
+    // Mark the directory as writable so that we can delete its children
+    if (andDelete && fs::is_directory(fromStatus)) {
+        fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+    }
+
+
+    if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
+        fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
+    } else if (fs::is_directory(fromStatus)) {
+        fs::create_directory(to);
+        for (auto & entry : fs::directory_iterator(from.path())) {
+            copy(entry, to / entry.path().filename(), andDelete);
+        }
+    } else {
+        throw Error("file '%s' has an unsupported type", from.path());
+    }
+
+    setWriteTime(to, statOfFrom);
+    if (andDelete) {
+        if (!fs::is_symlink(fromStatus))
+            fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+        fs::remove(from.path());
+    }
+}
+
+void renameFile(const Path & oldName, const Path & newName)
+{
+    fs::rename(oldName, newName);
+}
+
+void moveFile(const Path & oldName, const Path & newName)
+{
+    try {
+        renameFile(oldName, newName);
+    } catch (fs::filesystem_error & e) {
+        auto oldPath = fs::path(oldName);
+        auto newPath = fs::path(newName);
+        // For the move to be as atomic as possible, copy to a temporary
+        // directory
+        fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
+        Finally removeTemp = [&]() { fs::remove(temp); };
+        auto tempCopyTarget = temp / "copy-target";
+        if (e.code().value() == EXDEV) {
+            fs::remove(newPath);
+            warn("Can’t rename %s as %s, copying instead", oldName, newName);
+            copy(fs::directory_entry(oldPath), tempCopyTarget, true);
+            renameFile(tempCopyTarget, newPath);
+        }
+    }
+}
+
+}

src/libutil/git.cc

@@ -0,0 +1,25 @@
+#include "git.hh"
+
+#include <regex>
+
+namespace nix {
+namespace git {
+
+std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line)
+{
+    const static std::regex line_regex("^(ref: *)?([^\\s]+)(?:\\t+(.*))?$");
+    std::match_results<std::string_view::const_iterator> match;
+    if (!std::regex_match(line.cbegin(), line.cend(), match, line_regex))
+        return std::nullopt;
+
+    return LsRemoteRefLine {
+        .kind = match[1].length() == 0
+            ? LsRemoteRefLine::Kind::Object
+            : LsRemoteRefLine::Kind::Symbolic,
+        .target = match[2],
+        .reference = match[3].length() == 0 ? std::nullopt : std::optional<std::string>{ match[3] }
+    };
+}
+
+}
+}

src/libutil/git.hh

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <optional>
+
+namespace nix {
+
+namespace git {
+
+// A line from the output of `git ls-remote --symref`.
+//
+// These can be of two kinds:
+//
+// - Symbolic references of the form
+//
+//      ref: {target}	{reference}
+//
+//    where {target} is itself a reference and {reference} is optional
+//
+// - Object references of the form
+//
+//      {target}	{reference}
+//
+//    where {target} is a commit id and {reference} is mandatory
+struct LsRemoteRefLine {
+    enum struct Kind {
+        Symbolic,
+        Object
+    };
+    Kind kind;
+    std::string target;
+    std::optional<std::string> reference;
+};
+
+std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line);
+
+}
+
+}

src/libutil/hilite.cc

@@ -8,9 +8,9 @@ std::string hiliteMatches(
     std::string_view prefix,
     std::string_view postfix)
 {
-    // Avoid copy on zero matches
+    // Avoid extra work on zero matches
     if (matches.size() == 0)
-        return (std::string) s;
+        return std::string(s);
 
     std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) {
         return a.position() < b.position();

src/libutil/json-utils.hh

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+const nlohmann::json * get(const nlohmann::json & map, const std::string & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &*i;
+}
+
+nlohmann::json * get(nlohmann::json & map, const std::string & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &*i;
+}
+
+}

src/libutil/json.cc

@@ -1,11 +1,13 @@
 #include "json.hh"
 
 #include <iomanip>
+#include <cstdint>
 #include <cstring>
 
 namespace nix {
 
-void toJSON(std::ostream & str, const char * start, const char * end)
+template<>
+void toJSON<std::string_view>(std::ostream & str, const std::string_view & s)
 {
     constexpr size_t BUF_SIZE = 4096;
     char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts
@@ -20,7 +22,7 @@ void toJSON(std::ostream & str, const char * start, const char * end)
     };
 
     put('"');
-    for (auto i = start; i != end; i++) {
+    for (auto i = s.begin(); i != s.end(); i++) {
         if (bufPos >= BUF_SIZE) flush();
         if (*i == '\"' || *i == '\\') { put('\\'); put(*i); }
         else if (*i == '\n') { put('\\'); put('n'); }
@@ -43,7 +45,7 @@ void toJSON(std::ostream & str, const char * start, const char * end)
 
 void toJSON(std::ostream & str, const char * s)
 {
-    if (!s) str << "null"; else toJSON(str, s, s + strlen(s));
+    if (!s) str << "null"; else toJSON(str, std::string_view(s));
 }
 
 template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; }
@@ -54,11 +56,7 @@ template<> void toJSON<long long>(std::ostream & str, const long long & n) { str
 template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; }
 template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; }
 template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; }
-
-template<> void toJSON<std::string>(std::ostream & str, const std::string & s)
-{
-    toJSON(str, s.c_str(), s.c_str() + s.size());
-}
+template<> void toJSON<std::string>(std::ostream & str, const std::string & s) { toJSON(str, (std::string_view) s); }
 
 template<> void toJSON<bool>(std::ostream & str, const bool & b)
 {
@@ -153,7 +151,7 @@ JSONObject::~JSONObject()
     }
 }
 
-void JSONObject::attr(const std::string & s)
+void JSONObject::attr(std::string_view s)
 {
     comma();
     toJSON(state->str, s);
@@ -161,19 +159,19 @@ void JSONObject::attr(const std::string & s)
     if (state->indent) state->str << ' ';
 }
 
-JSONList JSONObject::list(const std::string & name)
+JSONList JSONObject::list(std::string_view name)
 {
     attr(name);
     return JSONList(state);
 }
 
-JSONObject JSONObject::object(const std::string & name)
+JSONObject JSONObject::object(std::string_view name)
 {
     attr(name);
     return JSONObject(state);
 }
 
-JSONPlaceholder JSONObject::placeholder(const std::string & name)
+JSONPlaceholder JSONObject::placeholder(std::string_view name)
 {
     attr(name);
     return JSONPlaceholder(state);
@@ -195,7 +193,11 @@ JSONObject JSONPlaceholder::object()
 
 JSONPlaceholder::~JSONPlaceholder()
 {
-    assert(!first || std::uncaught_exceptions());
+    if (first) {
+        assert(std::uncaught_exceptions());
+        if (state->stack != 0)
+            write(nullptr);
+    }
 }
 
 }

src/libutil/json.hh

@@ -6,7 +6,6 @@
 
 namespace nix {
 
-void toJSON(std::ostream & str, const char * start, const char * end);
 void toJSON(std::ostream & str, const char * s);
 
 template<typename T>
@@ -107,7 +106,7 @@ private:
         open();
     }
 
-    void attr(const std::string & s);
+    void attr(std::string_view s);
 
 public:
 
@@ -128,18 +127,18 @@ public:
     ~JSONObject();
 
     template<typename T>
-    JSONObject & attr(const std::string & name, const T & v)
+    JSONObject & attr(std::string_view name, const T & v)
     {
         attr(name);
         toJSON(state->str, v);
         return *this;
     }
 
-    JSONList list(const std::string & name);
+    JSONList list(std::string_view name);
 
-    JSONObject object(const std::string & name);
+    JSONObject object(std::string_view name);
 
-    JSONPlaceholder placeholder(const std::string & name);
+    JSONPlaceholder placeholder(std::string_view name);
 };
 
 class JSONPlaceholder : JSONWriter

src/libutil/logging.hh

@@ -111,6 +111,9 @@ public:
 
     virtual std::optional<char> ask(std::string_view s)
     { return {}; }
+
+    virtual void setPrintBuildLogs(bool printBuildLogs)
+    { }
 };
 
 ActivityId getCurActivity();

src/libutil/ref.hh

@@ -7,7 +7,7 @@
 namespace nix {
 
 /* A simple non-nullable reference-counted pointer. Actually a wrapper
-   around std::shared_ptr that prevents non-null constructions. */
+   around std::shared_ptr that prevents null constructions. */
 template<typename T>
 class ref
 {

src/libutil/serialise.cc

@@ -48,24 +48,9 @@ FdSink::~FdSink()
 }
 
 
-size_t threshold = 256 * 1024 * 1024;
-
-static void warnLargeDump()
-{
-    warn("dumping very large path (> 256 MiB); this may run out of memory");
-}
-
-
 void FdSink::write(std::string_view data)
 {
     written += data.size();
-    static bool warned = false;
-    if (warn && !warned) {
-        if (written > threshold) {
-            warnLargeDump();
-            warned = true;
-        }
-    }
     try {
         writeFull(fd, data);
     } catch (SysError & e) {
@@ -448,11 +433,6 @@ Error readError(Source & source)
 
 void StringSink::operator () (std::string_view data)
 {
-    static bool warned = false;
-    if (!warned && s.size() > threshold) {
-        warnLargeDump();
-        warned = true;
-    }
     s.append(data);
 }
 

src/libutil/serialise.hh

@@ -97,19 +97,17 @@ protected:
 struct FdSink : BufferedSink
 {
     int fd;
-    bool warn = false;
     size_t written = 0;
 
     FdSink() : fd(-1) { }
     FdSink(int fd) : fd(fd) { }
     FdSink(FdSink&&) = default;
 
-    FdSink& operator=(FdSink && s)
+    FdSink & operator=(FdSink && s)
     {
         flush();
         fd = s.fd;
         s.fd = -1;
-        warn = s.warn;
         written = s.written;
         return *this;
     }

src/libutil/tests/git.cc

@@ -0,0 +1,33 @@
+#include "git.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    TEST(GitLsRemote, parseSymrefLineWithReference) {
+        auto line = "ref: refs/head/main	HEAD";
+        auto res = git::parseLsRemoteLine(line);
+        ASSERT_TRUE(res.has_value());
+        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic);
+        ASSERT_EQ(res->target, "refs/head/main");
+        ASSERT_EQ(res->reference, "HEAD");
+    }
+
+    TEST(GitLsRemote, parseSymrefLineWithNoReference) {
+        auto line = "ref: refs/head/main";
+        auto res = git::parseLsRemoteLine(line);
+        ASSERT_TRUE(res.has_value());
+        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic);
+        ASSERT_EQ(res->target, "refs/head/main");
+        ASSERT_EQ(res->reference, std::nullopt);
+    }
+
+    TEST(GitLsRemote, parseObjectRefLine) {
+        auto line = "abc123	refs/head/main";
+        auto res = git::parseLsRemoteLine(line);
+        ASSERT_TRUE(res.has_value());
+        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object);
+        ASSERT_EQ(res->target, "abc123");
+        ASSERT_EQ(res->reference, "refs/head/main");
+    }
+}
+

src/libutil/tests/json.cc

@@ -102,8 +102,8 @@ namespace nix {
 
     TEST(toJSON, substringEscape) {
         std::stringstream out;
-        const char *s = "foo\t";
-        toJSON(out, s+3, s + strlen(s));
+        std::string_view s = "foo\t";
+        toJSON(out, s.substr(3));
 
         ASSERT_EQ(out.str(), "\"\\t\"");
     }

src/libutil/tests/tests.cc

@@ -548,7 +548,7 @@ namespace nix {
 
     TEST(get, emptyContainer) {
         StringMap s = { };
-        auto expected = std::nullopt;
+        auto expected = nullptr;
 
         ASSERT_EQ(get(s, "one"), expected);
     }
@@ -559,7 +559,23 @@ namespace nix {
         s["two"] = "er";
         auto expected = "yi";
 
-        ASSERT_EQ(get(s, "one"), expected);
+        ASSERT_EQ(*get(s, "one"), expected);
+    }
+
+    TEST(getOr, emptyContainer) {
+        StringMap s = { };
+        auto expected = "yi";
+
+        ASSERT_EQ(getOr(s, "one", "yi"), expected);
+    }
+
+    TEST(getOr, getFromContainer) {
+        StringMap s;
+        s["one"] = "yi";
+        s["two"] = "er";
+        auto expected = "yi";
+
+        ASSERT_EQ(getOr(s, "one", "nope"), expected);
     }
 
     /* ----------------------------------------------------------------------------

src/libutil/url.cc

@@ -1,6 +1,7 @@
 #include "url.hh"
 #include "url-parts.hh"
 #include "util.hh"
+#include "split.hh"
 
 namespace nix {
 
@@ -136,4 +137,21 @@ bool ParsedURL::operator ==(const ParsedURL & other) const
         && fragment == other.fragment;
 }
 
+/**
+ * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme'
+ * into a tuple '(applicationScheme, transportScheme)'
+ *
+ * > parseUrlScheme("http") == ParsedUrlScheme{ {}, "http"}
+ * > parseUrlScheme("tarball+http") == ParsedUrlScheme{ {"tarball"}, "http"}
+ */
+ParsedUrlScheme parseUrlScheme(std::string_view scheme)
+{
+    auto application = splitPrefixTo(scheme, '+');
+    auto transport = scheme;
+    return ParsedUrlScheme {
+        .application = application,
+        .transport = transport,
+    };
+}
+
 }

src/libutil/url.hh

@@ -27,4 +27,19 @@ std::map<std::string, std::string> decodeQuery(const std::string & query);
 
 ParsedURL parseURL(const std::string & url);
 
+/*
+ * Although that’s not really standardized anywhere, an number of tools
+ * use a scheme of the form 'x+y' in urls, where y is the “transport layer”
+ * scheme, and x is the “application layer” scheme.
+ *
+ * For example git uses `git+https` to designate remotes using a Git
+ * protocol over http.
+ */
+struct ParsedUrlScheme {
+    std::optional<std::string_view> application;
+    std::string_view transport;
+};
+
+ParsedUrlScheme parseUrlScheme(std::string_view scheme);
+
 }

src/libutil/util.cc

@@ -29,11 +29,15 @@
 
 #ifdef __APPLE__
 #include <sys/syscall.h>
+#include <mach-o/dyld.h>
 #endif
 
 #ifdef __linux__
 #include <sys/prctl.h>
 #include <sys/resource.h>
+
+#include <mntent.h>
+#include <cmath>
 #endif
 
 
@@ -504,61 +508,6 @@ void deletePath(const Path & path, uint64_t & bytesFreed)
 }
 
 
-static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
-    int & counter)
-{
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
-    if (includePid)
-        return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
-    else
-        return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str();
-}
-
-
-Path createTempDir(const Path & tmpRoot, const Path & prefix,
-    bool includePid, bool useGlobalCounter, mode_t mode)
-{
-    static int globalCounter = 0;
-    int localCounter = 0;
-    int & counter(useGlobalCounter ? globalCounter : localCounter);
-
-    while (1) {
-        checkInterrupt();
-        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
-        if (mkdir(tmpDir.c_str(), mode) == 0) {
-#if __FreeBSD__
-            /* Explicitly set the group of the directory.  This is to
-               work around around problems caused by BSD's group
-               ownership semantics (directories inherit the group of
-               the parent).  For instance, the group of /tmp on
-               FreeBSD is "wheel", so all directories created in /tmp
-               will be owned by "wheel"; but if the user is not in
-               "wheel", then "tar" will fail to unpack archives that
-               have the setgid bit set on directories. */
-            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
-                throw SysError("setting group of directory '%1%'", tmpDir);
-#endif
-            return tmpDir;
-        }
-        if (errno != EEXIST)
-            throw SysError("creating directory '%1%'", tmpDir);
-    }
-}
-
-
-std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
-{
-    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
-    // Strictly speaking, this is UB, but who cares...
-    // FIXME: use O_TMPFILE.
-    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
-    if (!fd)
-        throw SysError("creating temporary file '%s'", tmpl);
-    closeOnExec(fd.get());
-    return {std::move(fd), tmpl};
-}
-
-
 std::string getUserName()
 {
     auto pw = getpwuid(geteuid());
@@ -573,7 +522,21 @@ Path getHome()
 {
     static Path homeDir = []()
     {
+        std::optional<std::string> unownedUserHomeDir = {};
         auto homeDir = getEnv("HOME");
+        if (homeDir) {
+            // Only use $HOME if doesn't exist or is owned by the current user.
+            struct stat st;
+            int result = stat(homeDir->c_str(), &st);
+            if (result != 0) {
+                if (errno != ENOENT) {
+                    warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno);
+                    homeDir.reset();
+                }
+            } else if (st.st_uid != geteuid()) {
+                unownedUserHomeDir.swap(homeDir);
+            }
+        }
         if (!homeDir) {
             std::vector<char> buf(16384);
             struct passwd pwbuf;
@@ -582,6 +545,9 @@ Path getHome()
                 || !pw || !pw->pw_dir || !pw->pw_dir[0])
                 throw Error("cannot determine user's home directory");
             homeDir = pw->pw_dir;
+            if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
+                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
+            }
         }
         return *homeDir;
     }();
@@ -619,6 +585,27 @@ Path getDataDir()
 }
 
 
+std::optional<Path> getSelfExe()
+{
+    static auto cached = []() -> std::optional<Path>
+    {
+        #if __linux__
+        return readLink("/proc/self/exe");
+        #elif __APPLE__
+        char buf[1024];
+        uint32_t size = sizeof(buf);
+        if (_NSGetExecutablePath(buf, &size) == 0)
+            return buf;
+        else
+            return std::nullopt;
+        #else
+        return std::nullopt;
+        #endif
+    }();
+    return cached;
+}
+
+
 Paths createDirs(const Path & path)
 {
     Paths created;
@@ -642,44 +629,6 @@ Paths createDirs(const Path & path)
 }
 
 
-void createSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime)
-{
-    if (symlink(target.c_str(), link.c_str()))
-        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
-    if (mtime) {
-        struct timeval times[2];
-        times[0].tv_sec = *mtime;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = *mtime;
-        times[1].tv_usec = 0;
-        if (lutimes(link.c_str(), times))
-            throw SysError("setting time of symlink '%s'", link);
-    }
-}
-
-
-void replaceSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime)
-{
-    for (unsigned int n = 0; true; n++) {
-        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
-
-        try {
-            createSymlink(target, tmp, mtime);
-        } catch (SysError & e) {
-            if (e.errNo == EEXIST) continue;
-            throw;
-        }
-
-        if (rename(tmp.c_str(), link.c_str()) != 0)
-            throw SysError("renaming '%1%' to '%2%'", tmp, link);
-
-        break;
-    }
-}
-
-
 void readFull(int fd, char * buf, size_t count)
 {
     while (count) {
@@ -752,7 +701,55 @@ void drainFD(int fd, Sink & sink, bool block)
     }
 }
 
+//////////////////////////////////////////////////////////////////////
 
+unsigned int getMaxCPU()
+{
+    #if __linux__
+    try {
+        FILE *fp = fopen("/proc/mounts", "r");
+        if (!fp)
+            return 0;
+
+        Strings cgPathParts;
+
+        struct mntent *ent;
+        while ((ent = getmntent(fp))) {
+            std::string mountType, mountPath;
+
+            mountType = ent->mnt_type;
+            mountPath = ent->mnt_dir;
+
+            if (mountType == "cgroup2") {
+                cgPathParts.push_back(mountPath);
+                break;
+            }
+        }
+
+        fclose(fp);
+
+        if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) {
+            std::string currentCgroup = readFile("/proc/self/cgroup");
+            Strings cgValues = tokenizeString<Strings>(currentCgroup, ":");
+            cgPathParts.push_back(trim(cgValues.back(), "\n"));
+            cgPathParts.push_back("cpu.max");
+            std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts));
+
+            if (pathExists(fullCgPath)) {
+                std::string cpuMax = readFile(fullCgPath);
+                std::vector<std::string> cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " ");
+                std::string quota = cpuMaxParts[0];
+                std::string period = trim(cpuMaxParts[1], "\n");
+
+                if (quota != "max")
+                    return std::ceil(std::stoi(quota) / std::stof(period));
+            }
+        }
+    } catch (Error &) { ignoreException(); }
+    #endif
+
+    return 0;
+}
 
 //////////////////////////////////////////////////////////////////////
 
@@ -1588,7 +1585,6 @@ std::string stripIndentation(std::string_view s)
 
 //////////////////////////////////////////////////////////////////////
 
-
 static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
 
 
@@ -1819,7 +1815,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
     if (chmod(path.c_str(), mode) == -1)
         throw SysError("changing permissions on '%1%'", path);
 
-    if (listen(fdSocket.get(), 5) == -1)
+    if (listen(fdSocket.get(), 100) == -1)
         throw SysError("cannot listen on socket '%1%'", path);
 
     return fdSocket;

src/libutil/util.hh

@@ -149,10 +149,14 @@ std::vector<Path> getConfigDirs();
 /* Return $XDG_DATA_HOME or $HOME/.local/share. */
 Path getDataDir();
 
+/* Return the path of the current executable. */
+std::optional<Path> getSelfExe();
+
 /* Create a directory and all its parents, if necessary.  Returns the
    list of created directories, in order of creation. */
 Paths createDirs(const Path & path);
-inline Paths createDirs(PathView path) {
+inline Paths createDirs(PathView path)
+{
     return createDirs(Path(path));
 }
 
@@ -164,6 +168,17 @@ void createSymlink(const Path & target, const Path & link,
 void replaceSymlink(const Path & target, const Path & link,
     std::optional<time_t> mtime = {});
 
+void renameFile(const Path & src, const Path & dst);
+
+/**
+ * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
+ * are on a different filesystem.
+ *
+ * Beware that this might not be atomic because of the copy that happens behind
+ * the scenes
+ */
+void moveFile(const Path & src, const Path & dst);
+
 
 /* Wrappers arount read()/write() that read/write exactly the
    requested number of bytes. */
@@ -178,6 +193,9 @@ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
 
 void drainFD(int fd, Sink & sink, bool block = true);
 
+/* If cgroups are active, attempt to calculate the number of CPUs available.
+   If cgroups are unavailable or if cpu.max is set to "max", return 0. */
+unsigned int getMaxCPU();
 
 /* Automatic cleanup of resources. */
 
@@ -543,13 +561,31 @@ std::string stripIndentation(std::string_view s);
 
 /* Get a value for the specified key from an associate container. */
 template <class T>
-std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
+const typename T::mapped_type * get(const T & map, const typename T::key_type & key)
 {
     auto i = map.find(key);
-    if (i == map.end()) return {};
-    return std::optional<typename T::mapped_type>(i->second);
+    if (i == map.end()) return nullptr;
+    return &i->second;
 }
 
+template <class T>
+typename T::mapped_type * get(T & map, const typename T::key_type & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &i->second;
+}
+
+/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */
+template <class T>
+const typename T::mapped_type & getOr(T & map,
+    const typename T::key_type & key,
+    const typename T::mapped_type & defaultValue)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return defaultValue;
+    return i->second;
+}
 
 /* Remove and return the first item from a container. */
 template <class T>
@@ -682,4 +718,19 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 std::string showBytes(uint64_t bytes);
 
 
+/* Provide an addition operator between strings and string_views
+   inexplicably omitted from the standard library. */
+inline std::string operator + (const std::string & s1, std::string_view s2)
+{
+    auto s = s1;
+    s.append(s2);
+    return s;
+}
+
+inline std::string operator + (std::string && s, std::string_view s2)
+{
+    s.append(s2);
+    return std::move(s);
+}
+
 }