Merge commit 'f66bbd8c7b' into auto-uid-allocation

src/build-remote/build-remote.cc

@@ -33,18 +33,18 @@ std::string escapeUri(std::string uri)
 
 static string currentLoad;
 
-static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
+static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
 {
     return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
 }
 
-static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
+static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
     for (auto & feature : requiredFeatures)
-        if (!settings.systemFeatures.get().count(feature)) return false;
+        if (!store.systemFeatures.get().count(feature)) return false;
     return true;
 }
 
-static int _main(int argc, char * * argv)
+static int main_build_remote(int argc, char * * argv)
 {
     {
         logger = makeJSONLogger(*logger);
@@ -103,10 +103,10 @@ static int _main(int argc, char * * argv)
             drvPath = store->parseStorePath(readString(source));
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
-             auto canBuildLocally = amWilling
+            auto canBuildLocally = amWilling
                  &&  (  neededSystem == settings.thisSystem
                      || settings.extraPlatforms.get().count(neededSystem) > 0)
-                 &&  allSupportedLocally(requiredFeatures);
+                 &&  allSupportedLocally(*store, requiredFeatures);
 
             /* Error ignored here, will be caught later */
             mkdir(currentLoad.c_str(), 0777);
@@ -119,7 +119,7 @@ static int _main(int argc, char * * argv)
                 bool rightType = false;
 
                 Machine * bestMachine = nullptr;
-                unsigned long long bestLoad = 0;
+                uint64_t bestLoad = 0;
                 for (auto & m : machines) {
                     debug("considering building on remote machine '%s'", m.storeUri);
 
@@ -130,8 +130,8 @@ static int _main(int argc, char * * argv)
                         m.mandatoryMet(requiredFeatures)) {
                         rightType = true;
                         AutoCloseFD free;
-                        unsigned long long load = 0;
-                        for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) {
+                        uint64_t load = 0;
+                        for (uint64_t slot = 0; slot < m.maxJobs; ++slot) {
                             auto slotLock = openSlotLock(m, slot);
                             if (lockFile(slotLock.get(), ltWrite, false)) {
                                 if (!free) {
@@ -170,7 +170,45 @@ static int _main(int argc, char * * argv)
                     if (rightType && !canBuildLocally)
                         std::cerr << "# postpone\n";
                     else
+                    {
+                        // build the hint template.
+                        string hintstring =  "derivation: %s\nrequired (system, features): (%s, %s)";
+                        hintstring += "\n%s available machines:";
+                        hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
+
+                        for (unsigned int i = 0; i < machines.size(); ++i) {
+                          hintstring += "\n(%s, %s, %s, %s)";
+                        }
+
+                        // add the template values.
+                        string drvstr;
+                        if (drvPath.has_value())
+                            drvstr = drvPath->to_string();
+                        else
+                            drvstr = "<unknown>";
+
+                        auto hint = hintformat(hintstring);
+                        hint
+                          % drvstr
+                          % neededSystem
+                          % concatStringsSep<StringSet>(", ", requiredFeatures)
+                          % machines.size();
+
+                        for (auto & m : machines) {
+                          hint % concatStringsSep<vector<string>>(", ", m.systemTypes)
+                            % m.maxJobs
+                            % concatStringsSep<StringSet>(", ", m.supportedFeatures)
+                            % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
+                        }
+
+                        logErrorInfo(lvlInfo, {
+                              .name = "Remote build",
+                              .description = "Failed to find a machine for remote build!",
+                              .hint = hint
+                        });
+
                         std::cerr << "# decline\n";
+                    }
                     break;
                 }
 
@@ -186,15 +224,7 @@ static int _main(int argc, char * * argv)
 
                     Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
 
-                    Store::Params storeParams;
-                    if (hasPrefix(bestMachine->storeUri, "ssh://")) {
-                        storeParams["max-connections"] = "1";
-                        storeParams["log-fd"] = "4";
-                        if (bestMachine->sshKey != "")
-                            storeParams["ssh-key"] = bestMachine->sshKey;
-                    }
-
-                    sshStore = openStore(bestMachine->storeUri, storeParams);
+                    sshStore = bestMachine->openStore();
                     sshStore->connect();
                     storeUri = bestMachine->storeUri;
 
@@ -267,4 +297,4 @@ connected:
     }
 }
 
-static RegisterLegacyCommand s1("build-remote", _main);
+static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote);

src/libexpr/common-eval-args.cc

@@ -4,6 +4,8 @@
 #include "util.hh"
 #include "eval.hh"
 #include "fetchers.hh"
+#include "registry.hh"
+#include "flake/flakeref.hh"
 #include "store-api.hh"
 
 namespace nix {
@@ -27,10 +29,31 @@ MixEvalArgs::MixEvalArgs()
     addFlag({
         .longName = "include",
         .shortName = 'I',
-        .description = "add a path to the list of locations used to look up <...> file names",
+        .description = "add a path to the list of locations used to look up `<...>` file names",
         .labels = {"path"},
         .handler = {[&](std::string s) { searchPath.push_back(s); }}
     });
+
+    addFlag({
+        .longName = "impure",
+        .description = "allow access to mutable paths and repositories",
+        .handler = {[&]() {
+            evalSettings.pureEval = false;
+        }},
+    });
+
+    addFlag({
+        .longName = "override-flake",
+        .description = "override a flake registry value",
+        .labels = {"original-ref", "resolved-ref"},
+        .handler = {[&](std::string _from, std::string _to) {
+            auto from = parseFlakeRef(_from, absPath("."));
+            auto to = parseFlakeRef(_to, absPath("."));
+            fetchers::Attrs extraAttrs;
+            if (to.subdir != "") extraAttrs["dir"] = to.subdir;
+            fetchers::overrideRegistry(from.input, to.input, extraAttrs);
+        }}
+    });
 }
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
@@ -53,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s)
     if (isUri(s)) {
         return state.store->toRealPath(
             fetchers::downloadTarball(
-                state.store, resolveUri(s), "source", false).storePath);
+                state.store, resolveUri(s), "source", false).first.storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);

src/libexpr/eval-cache.cc

@@ -0,0 +1,620 @@
+#include "eval-cache.hh"
+#include "sqlite.hh"
+#include "eval.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+
+namespace nix::eval_cache {
+
+static const char * schema = R"sql(
+create table if not exists Attributes (
+    parent      integer not null,
+    name        text,
+    type        integer not null,
+    value       text,
+    context     text,
+    primary key (parent, name)
+);
+)sql";
+
+struct AttrDb
+{
+    std::atomic_bool failed{false};
+
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt insertAttribute;
+        SQLiteStmt insertAttributeWithContext;
+        SQLiteStmt queryAttribute;
+        SQLiteStmt queryAttributes;
+        std::unique_ptr<SQLiteTxn> txn;
+    };
+
+    std::unique_ptr<Sync<State>> _state;
+
+    AttrDb(const Hash & fingerprint)
+        : _state(std::make_unique<Sync<State>>())
+    {
+        auto state(_state->lock());
+
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
+        createDirs(cacheDir);
+
+        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
+
+        state->db = SQLite(dbPath);
+        state->db.isCache();
+        state->db.exec(schema);
+
+        state->insertAttribute.create(state->db,
+            "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
+
+        state->insertAttributeWithContext.create(state->db,
+            "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
+
+        state->queryAttribute.create(state->db,
+            "select rowid, type, value, context from Attributes where parent = ? and name = ?");
+
+        state->queryAttributes.create(state->db,
+            "select name from Attributes where parent = ?");
+
+        state->txn = std::make_unique<SQLiteTxn>(state->db);
+    }
+
+    ~AttrDb()
+    {
+        try {
+            auto state(_state->lock());
+            if (!failed)
+                state->txn->commit();
+            state->txn.reset();
+        } catch (...) {
+            ignoreException();
+        }
+    }
+
+    template<typename F>
+    AttrId doSQLite(F && fun)
+    {
+        if (failed) return 0;
+        try {
+            return fun();
+        } catch (SQLiteError &) {
+            ignoreException();
+            failed = true;
+            return 0;
+        }
+    }
+
+    AttrId setAttrs(
+        AttrKey key,
+        const std::vector<Symbol> & attrs)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::FullAttrs)
+                (0, false).exec();
+
+            AttrId rowId = state->db.getLastInsertedRowId();
+            assert(rowId);
+
+            for (auto & attr : attrs)
+                state->insertAttribute.use()
+                    (rowId)
+                    (attr)
+                    (AttrType::Placeholder)
+                    (0, false).exec();
+
+            return rowId;
+        });
+    }
+
+    AttrId setString(
+        AttrKey key,
+        std::string_view s,
+        const char * * context = nullptr)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            if (context) {
+                std::string ctx;
+                for (const char * * p = context; *p; ++p) {
+                    if (p != context) ctx.push_back(' ');
+                    ctx.append(*p);
+                }
+                state->insertAttributeWithContext.use()
+                    (key.first)
+                    (key.second)
+                    (AttrType::String)
+                    (s)
+                    (ctx).exec();
+            } else {
+                state->insertAttribute.use()
+                    (key.first)
+                    (key.second)
+                    (AttrType::String)
+                (s).exec();
+            }
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setBool(
+        AttrKey key,
+        bool b)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Bool)
+                (b ? 1 : 0).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setPlaceholder(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Placeholder)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setMissing(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Missing)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setMisc(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Misc)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setFailed(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Failed)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> getAttr(
+        AttrKey key,
+        SymbolTable & symbols)
+    {
+        auto state(_state->lock());
+
+        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
+        if (!queryAttribute.next()) return {};
+
+        auto rowId = (AttrType) queryAttribute.getInt(0);
+        auto type = (AttrType) queryAttribute.getInt(1);
+
+        switch (type) {
+            case AttrType::Placeholder:
+                return {{rowId, placeholder_t()}};
+            case AttrType::FullAttrs: {
+                // FIXME: expensive, should separate this out.
+                std::vector<Symbol> attrs;
+                auto queryAttributes(state->queryAttributes.use()(rowId));
+                while (queryAttributes.next())
+                    attrs.push_back(symbols.create(queryAttributes.getStr(0)));
+                return {{rowId, attrs}};
+            }
+            case AttrType::String: {
+                std::vector<std::pair<Path, std::string>> context;
+                if (!queryAttribute.isNull(3))
+                    for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
+                        context.push_back(decodeContext(s));
+                return {{rowId, string_t{queryAttribute.getStr(2), context}}};
+            }
+            case AttrType::Bool:
+                return {{rowId, queryAttribute.getInt(2) != 0}};
+            case AttrType::Missing:
+                return {{rowId, missing_t()}};
+            case AttrType::Misc:
+                return {{rowId, misc_t()}};
+            case AttrType::Failed:
+                return {{rowId, failed_t()}};
+            default:
+                throw Error("unexpected type in evaluation cache");
+        }
+    }
+};
+
+static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
+{
+    try {
+        return std::make_shared<AttrDb>(fingerprint);
+    } catch (SQLiteError &) {
+        ignoreException();
+        return nullptr;
+    }
+}
+
+EvalCache::EvalCache(
+    std::optional<std::reference_wrapper<const Hash>> useCache,
+    EvalState & state,
+    RootLoader rootLoader)
+    : db(useCache ? makeAttrDb(*useCache) : nullptr)
+    , state(state)
+    , rootLoader(rootLoader)
+{
+}
+
+Value * EvalCache::getRootValue()
+{
+    if (!value) {
+        debug("getting root value");
+        value = allocRootValue(rootLoader());
+    }
+    return *value;
+}
+
+std::shared_ptr<AttrCursor> EvalCache::getRoot()
+{
+    return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
+}
+
+AttrCursor::AttrCursor(
+    ref<EvalCache> root,
+    Parent parent,
+    Value * value,
+    std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
+    : root(root), parent(parent), cachedValue(std::move(cachedValue))
+{
+    if (value)
+        _value = allocRootValue(value);
+}
+
+AttrKey AttrCursor::getKey()
+{
+    if (!parent)
+        return {0, root->state.sEpsilon};
+    if (!parent->first->cachedValue) {
+        parent->first->cachedValue = root->db->getAttr(
+            parent->first->getKey(), root->state.symbols);
+        assert(parent->first->cachedValue);
+    }
+    return {parent->first->cachedValue->first, parent->second};
+}
+
+Value & AttrCursor::getValue()
+{
+    if (!_value) {
+        if (parent) {
+            auto & vParent = parent->first->getValue();
+            root->state.forceAttrs(vParent);
+            auto attr = vParent.attrs->get(parent->second);
+            if (!attr)
+                throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
+            _value = allocRootValue(attr->value);
+        } else
+            _value = allocRootValue(root->getRootValue());
+    }
+    return **_value;
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath() const
+{
+    if (parent) {
+        auto attrPath = parent->first->getAttrPath();
+        attrPath.push_back(parent->second);
+        return attrPath;
+    } else
+        return {};
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
+{
+    auto attrPath = getAttrPath();
+    attrPath.push_back(name);
+    return attrPath;
+}
+
+std::string AttrCursor::getAttrPathStr() const
+{
+    return concatStringsSep(".", getAttrPath());
+}
+
+std::string AttrCursor::getAttrPathStr(Symbol name) const
+{
+    return concatStringsSep(".", getAttrPath(name));
+}
+
+Value & AttrCursor::forceValue()
+{
+    debug("evaluating uncached attribute %s", getAttrPathStr());
+
+    auto & v = getValue();
+
+    try {
+        root->state.forceValue(v);
+    } catch (EvalError &) {
+        debug("setting '%s' to failed", getAttrPathStr());
+        if (root->db)
+            cachedValue = {root->db->setFailed(getKey()), failed_t()};
+        throw;
+    }
+
+    if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
+        if (v.type == tString)
+            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
+                           string_t{v.string.s, {}}};
+        else if (v.type == tPath)
+            cachedValue = {root->db->setString(getKey(), v.path), v.path};
+        else if (v.type == tBool)
+            cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+        else if (v.type == tAttrs)
+            ; // FIXME: do something?
+        else
+            cachedValue = {root->db->setMisc(getKey()), misc_t()};
+    }
+
+    return v;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+
+        if (cachedValue) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                for (auto & attr : *attrs)
+                    if (attr == name)
+                        return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
+                return nullptr;
+            } else if (std::get_if<placeholder_t>(&cachedValue->second)) {
+                auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
+                if (attr) {
+                    if (std::get_if<missing_t>(&attr->second))
+                        return nullptr;
+                    else if (std::get_if<failed_t>(&attr->second)) {
+                        if (forceErrors)
+                            debug("reevaluating failed cached attribute '%s'");
+                        else
+                            throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+                    } else
+                        return std::make_shared<AttrCursor>(root,
+                            std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
+                }
+                // Incomplete attrset, so need to fall thru and
+                // evaluate to see whether 'name' exists
+            } else
+                return nullptr;
+                //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tAttrs)
+        return nullptr;
+        //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    auto attr = v.attrs->get(name);
+
+    if (!attr) {
+        if (root->db) {
+            if (!cachedValue)
+                cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+            root->db->setMissing({cachedValue->first, name});
+        }
+        return nullptr;
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+        cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
+    }
+
+    return std::make_shared<AttrCursor>(
+        root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
+{
+    return maybeGetAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+{
+    auto p = maybeGetAttr(name, forceErrors);
+    if (!p)
+        throw Error("attribute '%s' does not exist", getAttrPathStr(name));
+    return p;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
+{
+    return getAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
+{
+    auto res = shared_from_this();
+    for (auto & attr : attrPath) {
+        res = res->maybeGetAttr(attr);
+        if (!res) return {};
+    }
+    return res;
+}
+
+std::string AttrCursor::getString()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto s = std::get_if<string_t>(&cachedValue->second)) {
+                debug("using cached string attribute '%s'", getAttrPathStr());
+                return s->first;
+            } else
+                throw TypeError("'%s' is not a string", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tString && v.type != tPath)
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+
+    return v.type == tString ? v.string.s : v.path;
+}
+
+string_t AttrCursor::getStringWithContext()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto s = std::get_if<string_t>(&cachedValue->second)) {
+                debug("using cached string attribute '%s'", getAttrPathStr());
+                return *s;
+            } else
+                throw TypeError("'%s' is not a string", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type == tString)
+        return {v.string.s, v.getContext()};
+    else if (v.type == tPath)
+        return {v.path, {}};
+    else
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+}
+
+bool AttrCursor::getBool()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto b = std::get_if<bool>(&cachedValue->second)) {
+                debug("using cached Boolean attribute '%s'", getAttrPathStr());
+                return *b;
+            } else
+                throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tBool)
+        throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+
+    return v.boolean;
+}
+
+std::vector<Symbol> AttrCursor::getAttrs()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                debug("using cached attrset attribute '%s'", getAttrPathStr());
+                return *attrs;
+            } else
+                throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type != tAttrs)
+        throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    std::vector<Symbol> attrs;
+    for (auto & attr : *getValue().attrs)
+        attrs.push_back(attr.name);
+    std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
+        return (const string &) a < (const string &) b;
+    });
+
+    if (root->db)
+        cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
+
+    return attrs;
+}
+
+bool AttrCursor::isDerivation()
+{
+    auto aType = maybeGetAttr("type");
+    return aType && aType->getString() == "derivation";
+}
+
+StorePath AttrCursor::forceDerivation()
+{
+    auto aDrvPath = getAttr(root->state.sDrvPath, true);
+    auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
+    if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
+        /* The eval cache contains 'drvPath', but the actual path has
+           been garbage-collected. So force it to be regenerated. */
+        aDrvPath->forceValue();
+        if (!root->state.store->isValidPath(drvPath))
+            throw Error("don't know how to recreate store derivation '%s'!",
+                root->state.store->printStorePath(drvPath));
+    }
+    return drvPath;
+}
+
+}

src/libexpr/eval-cache.hh

@@ -0,0 +1,123 @@
+#pragma once
+
+#include "sync.hh"
+#include "hash.hh"
+#include "eval.hh"
+
+#include <functional>
+#include <variant>
+
+namespace nix::eval_cache {
+
+MakeError(CachedEvalError, EvalError);
+
+struct AttrDb;
+class AttrCursor;
+
+class EvalCache : public std::enable_shared_from_this<EvalCache>
+{
+    friend class AttrCursor;
+
+    std::shared_ptr<AttrDb> db;
+    EvalState & state;
+    typedef std::function<Value *()> RootLoader;
+    RootLoader rootLoader;
+    RootValue value;
+
+    Value * getRootValue();
+
+public:
+
+    EvalCache(
+        std::optional<std::reference_wrapper<const Hash>> useCache,
+        EvalState & state,
+        RootLoader rootLoader);
+
+    std::shared_ptr<AttrCursor> getRoot();
+};
+
+enum AttrType {
+    Placeholder = 0,
+    FullAttrs = 1,
+    String = 2,
+    Missing = 3,
+    Misc = 4,
+    Failed = 5,
+    Bool = 6,
+};
+
+struct placeholder_t {};
+struct missing_t {};
+struct misc_t {};
+struct failed_t {};
+typedef uint64_t AttrId;
+typedef std::pair<AttrId, Symbol> AttrKey;
+typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+
+typedef std::variant<
+    std::vector<Symbol>,
+    string_t,
+    placeholder_t,
+    missing_t,
+    misc_t,
+    failed_t,
+    bool
+    > AttrValue;
+
+class AttrCursor : public std::enable_shared_from_this<AttrCursor>
+{
+    friend class EvalCache;
+
+    ref<EvalCache> root;
+    typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
+    Parent parent;
+    RootValue _value;
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue;
+
+    AttrKey getKey();
+
+    Value & getValue();
+
+public:
+
+    AttrCursor(
+        ref<EvalCache> root,
+        Parent parent,
+        Value * value = nullptr,
+        std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
+
+    std::vector<Symbol> getAttrPath() const;
+
+    std::vector<Symbol> getAttrPath(Symbol name) const;
+
+    std::string getAttrPathStr() const;
+
+    std::string getAttrPathStr(Symbol name) const;
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+
+    std::shared_ptr<AttrCursor> getAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
+
+    std::string getString();
+
+    string_t getStringWithContext();
+
+    bool getBool();
+
+    std::vector<Symbol> getAttrs();
+
+    bool isDerivation();
+
+    Value & forceValue();
+
+    /* Force creation of the .drv file in the Nix store. */
+    StorePath forceDerivation();
+};
+
+}

src/libexpr/eval.cc

@@ -87,6 +87,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con
             else if (*i == '\n') str << "\\n";
             else if (*i == '\r') str << "\\r";
             else if (*i == '\t') str << "\\t";
+            else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
             else str << *i;
         str << "\"";
         break;
@@ -199,6 +200,18 @@ string showType(const Value & v)
 }
 
 
+bool Value::isTrivial() const
+{
+    return
+        type != tApp
+        && type != tPrimOpApp
+        && (type != tThunk
+            || (dynamic_cast<ExprAttrs *>(thunk.expr)
+                && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
+            || dynamic_cast<ExprLambda *>(thunk.expr));
+}
+
+
 #if HAVE_BOEHMGC
 /* Called when the Boehm GC runs out of memory. */
 static void * oomHandler(size_t requested)
@@ -333,12 +346,17 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sStructuredAttrs(symbols.create("__structuredAttrs"))
     , sBuilder(symbols.create("builder"))
     , sArgs(symbols.create("args"))
+    , sContentAddressed(symbols.create("__contentAddressed"))
     , sOutputHash(symbols.create("outputHash"))
     , sOutputHashAlgo(symbols.create("outputHashAlgo"))
     , sOutputHashMode(symbols.create("outputHashMode"))
     , sRecurseForDerivations(symbols.create("recurseForDerivations"))
+    , sDescription(symbols.create("description"))
+    , sSelf(symbols.create("self"))
+    , sEpsilon(symbols.create(""))
     , repair(NoRepair)
     , store(store)
+    , regexCache(makeRegexCache())
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)
 {
@@ -353,7 +371,11 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
         for (auto & i : _searchPath) addToSearchPath(i);
         for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
     }
-    addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
+
+    try {
+        addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
+    } catch (Error &) {
+    }
 
     if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
@@ -365,10 +387,14 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
             auto path = r.second;
 
             if (store->isInStore(r.second)) {
-                StorePathSet closure;
-                store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
-                for (auto & path : closure)
-                    allowedPaths->insert(store->printStorePath(path));
+                try {
+                    StorePathSet closure;
+                    store->computeFSClosure(store->toStorePath(r.second).first, closure);
+                    for (auto & path : closure)
+                        allowedPaths->insert(store->printStorePath(path));
+                } catch (InvalidPath &) {
+                    allowedPaths->insert(r.second);
+                }
             } else
                 allowedPaths->insert(r.second);
         }
@@ -493,7 +519,7 @@ Value * EvalState::addPrimOp(const string & name,
     if (arity == 0) {
         auto vPrimOp = allocValue();
         vPrimOp->type = tPrimOp;
-        vPrimOp->primOp = new PrimOp(primOp, 1, sym);
+        vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym };
         Value v;
         mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(name, v);
@@ -501,7 +527,7 @@ Value * EvalState::addPrimOp(const string & name,
 
     Value * v = allocValue();
     v->type = tPrimOp;
-    v->primOp = new PrimOp(primOp, arity, sym);
+    v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym };
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
@@ -509,12 +535,59 @@ Value * EvalState::addPrimOp(const string & name,
 }
 
 
+Value * EvalState::addPrimOp(PrimOp && primOp)
+{
+    /* Hack to make constants lazy: turn them into a application of
+       the primop to a dummy value. */
+    if (primOp.arity == 0) {
+        primOp.arity = 1;
+        auto vPrimOp = allocValue();
+        vPrimOp->type = tPrimOp;
+        vPrimOp->primOp = new PrimOp(std::move(primOp));
+        Value v;
+        mkApp(v, *vPrimOp, *vPrimOp);
+        return addConstant(primOp.name, v);
+    }
+
+    Symbol envName = primOp.name;
+    if (hasPrefix(primOp.name, "__"))
+        primOp.name = symbols.create(std::string(primOp.name, 2));
+
+    Value * v = allocValue();
+    v->type = tPrimOp;
+    v->primOp = new PrimOp(std::move(primOp));
+    staticBaseEnv.vars[envName] = baseEnvDispl;
+    baseEnv.values[baseEnvDispl++] = v;
+    baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
+    return v;
+}
+
+
 Value & EvalState::getBuiltin(const string & name)
 {
     return *baseEnv.values[0]->attrs->find(symbols.create(name))->value;
 }
 
 
+std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
+{
+    if (v.type == tPrimOp || v.type == tPrimOpApp) {
+        auto v2 = &v;
+        while (v2->type == tPrimOpApp)
+            v2 = v2->primOpApp.left;
+        if (v2->primOp->doc)
+            return Doc {
+                .pos = noPos,
+                .name = v2->primOp->name,
+                .arity = v2->primOp->arity,
+                .args = v2->primOp->args,
+                .doc = v2->primOp->doc,
+            };
+    }
+    return {};
+}
+
+
 /* Every "format" object (even temporary) takes up a few hundred bytes
    of stack space, which is a real killer in the recursive
    evaluator.  So here are some helper functions for throwing
@@ -782,7 +855,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 }
 
 
-void EvalState::evalFile(const Path & path_, Value & v)
+void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
 {
     auto path = checkSourcePath(path_);
 
@@ -811,6 +884,11 @@ void EvalState::evalFile(const Path & path_, Value & v)
     fileParseCache[path2] = e;
 
     try {
+        // Enforce that 'flake.nix' is a direct attrset, not a
+        // computation.
+        if (mustBeTrivial &&
+            !(dynamic_cast<ExprAttrs *>(e)))
+            throw Error("file '%s' must be an attribute set", path);
         eval(e, v);
     } catch (Error & e) {
         addErrorTrace(e, "while evaluating the file '%1%':", path2);
@@ -1236,10 +1314,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
         try {
             lambda.body->eval(*this, env2, v);
         } catch (Error & e) {
-            addErrorTrace(e, lambda.pos, "while evaluating %s", 
-              (lambda.name.set() 
-                  ? "'" + (string) lambda.name + "'" 
-                  : "anonymous lambdaction"));
+            addErrorTrace(e, lambda.pos, "while evaluating %s",
+              (lambda.name.set()
+                  ? "'" + (string) lambda.name + "'"
+                  : "anonymous lambda"));
             addErrorTrace(e, pos, "from call site%s", "");
             throw;
         }
@@ -1276,14 +1354,25 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
     }
 
     Value * actualArgs = allocValue();
-    mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size());
+    mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size()));
 
-    for (auto & i : fun.lambda.fun->formals->formals) {
-        Bindings::iterator j = args.find(i.name);
-        if (j != args.end())
-            actualArgs->attrs->push_back(*j);
-        else if (!i.def)
-            throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
+    if (fun.lambda.fun->formals->ellipsis) {
+        // If the formals have an ellipsis (eg the function accepts extra args) pass
+        // all available automatic arguments (which includes arguments specified on
+        // the command line via --arg/--argstr)
+        for (auto& v : args) {
+            actualArgs->attrs->push_back(v);
+        }
+    } else {
+        // Otherwise, only pass the arguments that the function accepts
+        for (auto & i : fun.lambda.fun->formals->formals) {
+            Bindings::iterator j = args.find(i.name);
+            if (j != args.end()) {
+                actualArgs->attrs->push_back(*j);
+            } else if (!i.def) {
+                throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
+            }
+        }
     }
 
     actualArgs->attrs->sort();
@@ -1586,6 +1675,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
 }
 
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s)
+{
+    if (s.at(0) == '!') {
+        size_t index = s.find("!", 1);
+        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+    } else
+        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+}
+
+
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
@@ -1594,6 +1695,17 @@ void copyContext(const Value & v, PathSet & context)
 }
 
 
+std::vector<std::pair<Path, std::string>> Value::getContext()
+{
+    std::vector<std::pair<Path, std::string>> res;
+    assert(type == tString);
+    if (string.context)
+        for (const char * * p = string.context; *p; ++p)
+            res.push_back(decodeContext(*p));
+    return res;
+}
+
+
 string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
 {
     string s = forceString(v, pos);
@@ -1969,7 +2081,7 @@ Strings EvalSettings::getDefaultNixPath()
 
 EvalSettings evalSettings;
 
-static GlobalConfig::Register r1(&evalSettings);
+static GlobalConfig::Register rEvalSettings(&evalSettings);
 
 
 }

src/libexpr/eval.hh

@@ -4,13 +4,12 @@
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
-#include "hash.hh"
 #include "config.hh"
 
-#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>
+#include <mutex>
 
 
 namespace nix {
@@ -30,8 +29,8 @@ struct PrimOp
     PrimOpFun fun;
     size_t arity;
     Symbol name;
-    PrimOp(PrimOpFun fun, size_t arity, Symbol name)
-        : fun(fun), arity(arity), name(name) { }
+    std::vector<std::string> args;
+    const char * doc = nullptr;
 };
 
 
@@ -65,6 +64,11 @@ typedef std::list<SearchPathElem> SearchPath;
 void initGC();
 
 
+struct RegexCache;
+
+std::shared_ptr<RegexCache> makeRegexCache();
+
+
 class EvalState
 {
 public:
@@ -74,8 +78,10 @@ public:
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
+        sContentAddressed,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
-        sRecurseForDerivations;
+        sRecurseForDerivations,
+        sDescription, sSelf, sEpsilon;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -90,6 +96,7 @@ public:
 
     const ref<Store> store;
 
+
 private:
     SrcToStore srcToStore;
 
@@ -117,7 +124,7 @@ private:
     std::unordered_map<Path, Path> resolvedPaths;
 
     /* Cache used by prim_match(). */
-    std::unordered_map<std::string, std::regex> regexCache;
+    std::shared_ptr<RegexCache> regexCache;
 
 public:
 
@@ -152,8 +159,9 @@ public:
     Expr * parseStdin();
 
     /* Evaluate an expression read from the given file to normal
-       form. */
-    void evalFile(const Path & path, Value & v);
+       form. Optionally enforce that the top-level expression is
+       trivial (i.e. doesn't require arbitrary computation). */
+    void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
 
     void resetFileCache();
 
@@ -238,10 +246,23 @@ private:
     Value * addPrimOp(const string & name,
         size_t arity, PrimOpFun primOp);
 
+    Value * addPrimOp(PrimOp && primOp);
+
 public:
 
     Value & getBuiltin(const string & name);
 
+    struct Doc
+    {
+        Pos pos;
+        std::optional<Symbol> name;
+        size_t arity;
+        std::vector<std::string> args;
+        const char * doc;
+    };
+
+    std::optional<Doc> getDoc(Value & v);
+
 private:
 
     inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval);
@@ -330,7 +351,7 @@ string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
    name>. */
-std::pair<string, string> decodeContext(const string & s);
+std::pair<string, string> decodeContext(std::string_view s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
@@ -353,24 +374,60 @@ struct EvalSettings : Config
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 
-    Setting<Strings> nixPath{this, getDefaultNixPath(), "nix-path",
-        "List of directories to be searched for <...> file references."};
+    Setting<Strings> nixPath{
+        this, getDefaultNixPath(), "nix-path",
+        "List of directories to be searched for `<...>` file references."};
 
-    Setting<bool> restrictEval{this, false, "restrict-eval",
-        "Whether to restrict file system access to paths in $NIX_PATH, "
-        "and network access to the URI prefixes listed in 'allowed-uris'."};
+    Setting<bool> restrictEval{
+        this, false, "restrict-eval",
+        R"(
+          If set to `true`, the Nix evaluator will not allow access to any
+          files outside of the Nix search path (as set via the `NIX_PATH`
+          environment variable or the `-I` option), or to URIs outside of
+          `allowed-uri`. The default is `false`.
+        )"};
 
     Setting<bool> pureEval{this, false, "pure-eval",
         "Whether to restrict file system and network access to files specified by cryptographic hash."};
 
-    Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
-        "Whether the evaluator allows importing the result of a derivation."};
+    Setting<bool> enableImportFromDerivation{
+        this, true, "allow-import-from-derivation",
+        R"(
+          By default, Nix allows you to `import` from a derivation, allowing
+          building at evaluation time. With this option set to false, Nix will
+          throw an error when evaluating an expression that uses this feature,
+          allowing users to ensure their evaluation will not require any
+          builds to take place.
+        )"};
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
-        "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+        R"(
+          A list of URI prefixes to which access is allowed in restricted
+          evaluation mode. For example, when set to
+          `https://github.com/NixOS`, builtin functions such as `fetchGit` are
+          allowed to access `https://github.com/NixOS/patchelf.git`.
+        )"};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
+        R"(
+          If set to `true`, the Nix evaluator will trace every function call.
+          Nix will print a log message at the "vomit" level for every function
+          entrance and function exit.
+
+              function-trace entered undefined position at 1565795816999559622
+              function-trace exited undefined position at 1565795816999581277
+              function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
+              function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
+
+          The `undefined position` means the function call is a builtin.
+
+          Use the `contrib/stack-collapse.py` script distributed with the Nix
+          source code to convert the trace logs in to a format suitable for
+          `flamegraph.pl`.
+        )"};
+
+    Setting<bool> useEvalCache{this, true, "eval-cache",
+        "Whether to use the flake evaluation cache."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/flake/call-flake.nix

@@ -0,0 +1,56 @@
+lockFileStr: rootSrc: rootSubdir:
+
+let
+
+  lockFile = builtins.fromJSON lockFileStr;
+
+  allNodes =
+    builtins.mapAttrs
+      (key: node:
+        let
+
+          sourceInfo =
+            if key == lockFile.root
+            then rootSrc
+            else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
+
+          subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
+
+          flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
+
+          inputs = builtins.mapAttrs
+            (inputName: inputSpec: allNodes.${resolveInput inputSpec})
+            (node.inputs or {});
+
+          # Resolve a input spec into a node name. An input spec is
+          # either a node name, or a 'follows' path from the root
+          # node.
+          resolveInput = inputSpec:
+              if builtins.isList inputSpec
+              then getInputByPath lockFile.root inputSpec
+              else inputSpec;
+
+          # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
+          # root node, returning the final node.
+          getInputByPath = nodeName: path:
+            if path == []
+            then nodeName
+            else
+              getInputByPath
+                # Since this could be a 'follows' input, call resolveInput.
+                (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
+                (builtins.tail path);
+
+          outputs = flake.outputs (inputs // { self = result; });
+
+          result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; };
+        in
+          if node.flake or true then
+            assert builtins.isFunction flake.outputs;
+            result
+          else
+            sourceInfo
+      )
+      lockFile.nodes;
+
+in allNodes.${lockFile.root}

src/libexpr/flake/flake.cc

@@ -0,0 +1,608 @@
+#include "flake.hh"
+#include "lockfile.hh"
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+#include "fetchers.hh"
+#include "finally.hh"
+
+namespace nix {
+
+using namespace flake;
+
+namespace flake {
+
+typedef std::pair<fetchers::Tree, FlakeRef> FetchedFlake;
+typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
+
+static std::optional<FetchedFlake> lookupInFlakeCache(
+    const FlakeCache & flakeCache,
+    const FlakeRef & flakeRef)
+{
+    // FIXME: inefficient.
+    for (auto & i : flakeCache) {
+        if (flakeRef == i.first) {
+            debug("mapping '%s' to previously seen input '%s' -> '%s",
+                flakeRef, i.first, i.second.second);
+            return i.second;
+        }
+    }
+
+    return std::nullopt;
+}
+
+static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
+    EvalState & state,
+    const FlakeRef & originalRef,
+    bool allowLookup,
+    FlakeCache & flakeCache)
+{
+    auto fetched = lookupInFlakeCache(flakeCache, originalRef);
+    FlakeRef resolvedRef = originalRef;
+
+    if (!fetched) {
+        if (originalRef.input.isDirect()) {
+            fetched.emplace(originalRef.fetchTree(state.store));
+        } else {
+            if (allowLookup) {
+                resolvedRef = originalRef.resolve(state.store);
+                auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
+                if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
+                flakeCache.push_back({resolvedRef, *fetchedResolved});
+                fetched.emplace(*fetchedResolved);
+            }
+            else {
+                throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
+            }
+        }
+        flakeCache.push_back({originalRef, *fetched});
+    }
+
+    auto [tree, lockedRef] = *fetched;
+
+    debug("got tree '%s' from '%s'",
+        state.store->printStorePath(tree.storePath), lockedRef);
+
+    if (state.allowedPaths)
+        state.allowedPaths->insert(tree.actualPath);
+
+    assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
+
+    return {std::move(tree), resolvedRef, lockedRef};
+}
+
+static void expectType(EvalState & state, ValueType type,
+    Value & value, const Pos & pos)
+{
+    if (value.type == tThunk && value.isTrivial())
+        state.forceValue(value, pos);
+    if (value.type != type)
+        throw Error("expected %s but got %s at %s",
+            showType(type), showType(value.type), pos);
+}
+
+static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+    EvalState & state, Value * value, const Pos & pos);
+
+static FlakeInput parseFlakeInput(EvalState & state,
+    const std::string & inputName, Value * value, const Pos & pos)
+{
+    expectType(state, tAttrs, *value, pos);
+
+    FlakeInput input;
+
+    auto sInputs = state.symbols.create("inputs");
+    auto sUrl = state.symbols.create("url");
+    auto sFlake = state.symbols.create("flake");
+    auto sFollows = state.symbols.create("follows");
+
+    fetchers::Attrs attrs;
+    std::optional<std::string> url;
+
+    for (nix::Attr attr : *(value->attrs)) {
+        try {
+            if (attr.name == sUrl) {
+                expectType(state, tString, *attr.value, *attr.pos);
+                url = attr.value->string.s;
+                attrs.emplace("url", *url);
+            } else if (attr.name == sFlake) {
+                expectType(state, tBool, *attr.value, *attr.pos);
+                input.isFlake = attr.value->boolean;
+            } else if (attr.name == sInputs) {
+                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
+            } else if (attr.name == sFollows) {
+                expectType(state, tString, *attr.value, *attr.pos);
+                input.follows = parseInputPath(attr.value->string.s);
+            } else {
+                state.forceValue(*attr.value);
+                if (attr.value->type == tString)
+                    attrs.emplace(attr.name, attr.value->string.s);
+                else
+                    throw TypeError("flake input attribute '%s' is %s while a string is expected",
+                        attr.name, showType(*attr.value));
+            }
+        } catch (Error & e) {
+            e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name));
+            throw;
+        }
+    }
+
+    if (attrs.count("type"))
+        try {
+            input.ref = FlakeRef::fromAttrs(attrs);
+        } catch (Error & e) {
+            e.addTrace(pos, hintfmt("in flake input"));
+            throw;
+        }
+    else {
+        attrs.erase("url");
+        if (!attrs.empty())
+            throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
+        if (url)
+            input.ref = parseFlakeRef(*url, {}, true);
+    }
+
+    if (!input.follows && !input.ref)
+        input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}});
+
+    return input;
+}
+
+static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+    EvalState & state, Value * value, const Pos & pos)
+{
+    std::map<FlakeId, FlakeInput> inputs;
+
+    expectType(state, tAttrs, *value, pos);
+
+    for (nix::Attr & inputAttr : *(*value).attrs) {
+        inputs.emplace(inputAttr.name,
+            parseFlakeInput(state,
+                inputAttr.name,
+                inputAttr.value,
+                *inputAttr.pos));
+    }
+
+    return inputs;
+}
+
+static Flake getFlake(
+    EvalState & state,
+    const FlakeRef & originalRef,
+    bool allowLookup,
+    FlakeCache & flakeCache)
+{
+    auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+        state, originalRef, allowLookup, flakeCache);
+
+    // Guard against symlink attacks.
+    auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
+    if (!isInDir(flakeFile, sourceInfo.actualPath))
+        throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
+            lockedRef, state.store->printStorePath(sourceInfo.storePath));
+
+    Flake flake {
+        .originalRef = originalRef,
+        .resolvedRef = resolvedRef,
+        .lockedRef = lockedRef,
+        .sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
+    };
+
+    if (!pathExists(flakeFile))
+        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
+
+    Value vInfo;
+    state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
+
+    expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
+
+    auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
+
+    if (vInfo.attrs->get(sEdition))
+        warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
+
+    if (auto description = vInfo.attrs->get(state.sDescription)) {
+        expectType(state, tString, *description->value, *description->pos);
+        flake.description = description->value->string.s;
+    }
+
+    auto sInputs = state.symbols.create("inputs");
+
+    if (auto inputs = vInfo.attrs->get(sInputs))
+        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
+
+    auto sOutputs = state.symbols.create("outputs");
+
+    if (auto outputs = vInfo.attrs->get(sOutputs)) {
+        expectType(state, tLambda, *outputs->value, *outputs->pos);
+
+        if (outputs->value->lambda.fun->matchAttrs) {
+            for (auto & formal : outputs->value->lambda.fun->formals->formals) {
+                if (formal.name != state.sSelf)
+                    flake.inputs.emplace(formal.name, FlakeInput {
+                        .ref = parseFlakeRef(formal.name)
+                    });
+            }
+        }
+
+    } else
+        throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
+
+    for (auto & attr : *vInfo.attrs) {
+        if (attr.name != sEdition &&
+            attr.name != state.sDescription &&
+            attr.name != sInputs &&
+            attr.name != sOutputs)
+            throw Error("flake '%s' has an unsupported attribute '%s', at %s",
+                lockedRef, attr.name, *attr.pos);
+    }
+
+    return flake;
+}
+
+Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
+{
+    FlakeCache flakeCache;
+    return getFlake(state, originalRef, allowLookup, flakeCache);
+}
+
+/* Compute an in-memory lock file for the specified top-level flake,
+   and optionally write it to file, if the flake is writable. */
+LockedFlake lockFlake(
+    EvalState & state,
+    const FlakeRef & topRef,
+    const LockFlags & lockFlags)
+{
+    settings.requireExperimentalFeature("flakes");
+
+    FlakeCache flakeCache;
+
+    auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
+
+    // FIXME: symlink attack
+    auto oldLockFile = LockFile::read(
+        flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
+
+    debug("old lock file: %s", oldLockFile);
+
+    // FIXME: check whether all overrides are used.
+    std::map<InputPath, FlakeInput> overrides;
+    std::set<InputPath> overridesUsed, updatesUsed;
+
+    for (auto & i : lockFlags.inputOverrides)
+        overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
+
+    LockFile newLockFile;
+
+    std::vector<FlakeRef> parents;
+
+    std::function<void(
+        const FlakeInputs & flakeInputs,
+        std::shared_ptr<Node> node,
+        const InputPath & inputPathPrefix,
+        std::shared_ptr<const Node> oldNode)>
+        computeLocks;
+
+    computeLocks = [&](
+        const FlakeInputs & flakeInputs,
+        std::shared_ptr<Node> node,
+        const InputPath & inputPathPrefix,
+        std::shared_ptr<const Node> oldNode)
+    {
+        debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
+
+        /* Get the overrides (i.e. attributes of the form
+           'inputs.nixops.inputs.nixpkgs.url = ...'). */
+        // FIXME: check this
+        for (auto & [id, input] : flake.inputs) {
+            for (auto & [idOverride, inputOverride] : input.overrides) {
+                auto inputPath(inputPathPrefix);
+                inputPath.push_back(id);
+                inputPath.push_back(idOverride);
+                overrides.insert_or_assign(inputPath, inputOverride);
+            }
+        }
+
+        /* Go over the flake inputs, resolve/fetch them if
+           necessary (i.e. if they're new or the flakeref changed
+           from what's in the lock file). */
+        for (auto & [id, input2] : flakeInputs) {
+            auto inputPath(inputPathPrefix);
+            inputPath.push_back(id);
+            auto inputPathS = printInputPath(inputPath);
+            debug("computing input '%s'", inputPathS);
+
+            /* Do we have an override for this input from one of the
+               ancestors? */
+            auto i = overrides.find(inputPath);
+            bool hasOverride = i != overrides.end();
+            if (hasOverride) overridesUsed.insert(inputPath);
+            auto & input = hasOverride ? i->second : input2;
+
+            /* Resolve 'follows' later (since it may refer to an input
+               path we haven't processed yet. */
+            if (input.follows) {
+                InputPath target;
+                if (hasOverride || input.absolute)
+                    /* 'follows' from an override is relative to the
+                       root of the graph. */
+                    target = *input.follows;
+                else {
+                    /* Otherwise, it's relative to the current flake. */
+                    target = inputPathPrefix;
+                    for (auto & i : *input.follows) target.push_back(i);
+                }
+                debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
+                node->inputs.insert_or_assign(id, target);
+                continue;
+            }
+
+            assert(input.ref);
+
+            /* Do we have an entry in the existing lock file? And we
+               don't have a --update-input flag for this input? */
+            std::shared_ptr<LockedNode> oldLock;
+
+            updatesUsed.insert(inputPath);
+
+            if (oldNode && !lockFlags.inputUpdates.count(inputPath))
+                if (auto oldLock2 = get(oldNode->inputs, id))
+                    if (auto oldLock3 = std::get_if<0>(&*oldLock2))
+                        oldLock = *oldLock3;
+
+            if (oldLock
+                && oldLock->originalRef == *input.ref
+                && !hasOverride)
+            {
+                debug("keeping existing input '%s'", inputPathS);
+
+                /* Copy the input from the old lock since its flakeref
+                   didn't change and there is no override from a
+                   higher level flake. */
+                auto childNode = std::make_shared<LockedNode>(
+                    oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
+
+                node->inputs.insert_or_assign(id, childNode);
+
+                /* If we have an --update-input flag for an input
+                   of this input, then we must fetch the flake to
+                   update it. */
+                auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
+
+                auto hasChildUpdate =
+                    lb != lockFlags.inputUpdates.end()
+                    && lb->size() > inputPath.size()
+                    && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
+
+                if (hasChildUpdate) {
+                    auto inputFlake = getFlake(
+                        state, oldLock->lockedRef, false, flakeCache);
+                    computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
+                } else {
+                    /* No need to fetch this flake, we can be
+                       lazy. However there may be new overrides on the
+                       inputs of this flake, so we need to check
+                       those. */
+                    FlakeInputs fakeInputs;
+
+                    for (auto & i : oldLock->inputs) {
+                        if (auto lockedNode = std::get_if<0>(&i.second)) {
+                            fakeInputs.emplace(i.first, FlakeInput {
+                                .ref = (*lockedNode)->originalRef,
+                                .isFlake = (*lockedNode)->isFlake,
+                            });
+                        } else if (auto follows = std::get_if<1>(&i.second)) {
+                            fakeInputs.emplace(i.first, FlakeInput {
+                                .follows = *follows,
+                                .absolute = true
+                            });
+                        }
+                    }
+
+                    computeLocks(fakeInputs, childNode, inputPath, oldLock);
+                }
+
+            } else {
+                /* We need to create a new lock file entry. So fetch
+                   this input. */
+                debug("creating new input '%s'", inputPathS);
+
+                if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
+                    throw Error("cannot update flake input '%s' in pure mode", inputPathS);
+
+                if (input.isFlake) {
+                    auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
+
+                    /* Note: in case of an --override-input, we use
+                       the *original* ref (input2.ref) for the
+                       "original" field, rather than the
+                       override. This ensures that the override isn't
+                       nuked the next time we update the lock
+                       file. That is, overrides are sticky unless you
+                       use --no-write-lock-file. */
+                    auto childNode = std::make_shared<LockedNode>(
+                        inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
+
+                    node->inputs.insert_or_assign(id, childNode);
+
+                    /* Guard against circular flake imports. */
+                    for (auto & parent : parents)
+                        if (parent == *input.ref)
+                            throw Error("found circular import of flake '%s'", parent);
+                    parents.push_back(*input.ref);
+                    Finally cleanup([&]() { parents.pop_back(); });
+
+                    /* Recursively process the inputs of this
+                       flake. Also, unless we already have this flake
+                       in the top-level lock file, use this flake's
+                       own lock file. */
+                    computeLocks(
+                        inputFlake.inputs, childNode, inputPath,
+                        oldLock
+                        ? std::dynamic_pointer_cast<const Node>(oldLock)
+                        : LockFile::read(
+                            inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
+                }
+
+                else {
+                    auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+                        state, *input.ref, lockFlags.useRegistries, flakeCache);
+                    node->inputs.insert_or_assign(id,
+                        std::make_shared<LockedNode>(lockedRef, *input.ref, false));
+                }
+            }
+        }
+    };
+
+    computeLocks(
+        flake.inputs, newLockFile.root, {},
+        lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
+
+    for (auto & i : lockFlags.inputOverrides)
+        if (!overridesUsed.count(i.first))
+            warn("the flag '--override-input %s %s' does not match any input",
+                printInputPath(i.first), i.second);
+
+    for (auto & i : lockFlags.inputUpdates)
+        if (!updatesUsed.count(i))
+            warn("the flag '--update-input %s' does not match any input", printInputPath(i));
+
+    /* Check 'follows' inputs. */
+    newLockFile.check();
+
+    debug("new lock file: %s", newLockFile);
+
+    /* Check whether we need to / can write the new lock file. */
+    if (!(newLockFile == oldLockFile)) {
+
+        auto diff = LockFile::diff(oldLockFile, newLockFile);
+
+        if (lockFlags.writeLockFile) {
+            if (auto sourcePath = topRef.input.getSourcePath()) {
+                if (!newLockFile.isImmutable()) {
+                    if (settings.warnDirty)
+                        warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
+                } else {
+                    if (!lockFlags.updateLockFile)
+                        throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
+
+                    auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
+
+                    auto path = *sourcePath + "/" + relPath;
+
+                    bool lockFileExists = pathExists(path);
+
+                    if (lockFileExists) {
+                        auto s = chomp(diff);
+                        if (s.empty())
+                            warn("updating lock file '%s'", path);
+                        else
+                            warn("updating lock file '%s':\n%s", path, s);
+                    } else
+                        warn("creating lock file '%s'", path);
+
+                    newLockFile.write(path);
+
+                    topRef.input.markChangedFile(
+                        (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
+                        lockFlags.commitLockFile
+                        ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
+                                relPath, lockFileExists ? "Update" : "Add", diff))
+                        : std::nullopt);
+
+                    /* Rewriting the lockfile changed the top-level
+                       repo, so we should re-read it. FIXME: we could
+                       also just clear the 'rev' field... */
+                    auto prevLockedRef = flake.lockedRef;
+                    FlakeCache dummyCache;
+                    flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
+
+                    if (lockFlags.commitLockFile &&
+                        flake.lockedRef.input.getRev() &&
+                        prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
+                        warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
+
+                    /* Make sure that we picked up the change,
+                       i.e. the tree should usually be dirty
+                       now. Corner case: we could have reverted from a
+                       dirty to a clean tree! */
+                    if (flake.lockedRef.input == prevLockedRef.input
+                        && !flake.lockedRef.input.isImmutable())
+                        throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
+                }
+            } else
+                throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
+        } else
+            warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
+    }
+
+    return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
+}
+
+void callFlake(EvalState & state,
+    const LockedFlake & lockedFlake,
+    Value & vRes)
+{
+    auto vLocks = state.allocValue();
+    auto vRootSrc = state.allocValue();
+    auto vRootSubdir = state.allocValue();
+    auto vTmp1 = state.allocValue();
+    auto vTmp2 = state.allocValue();
+
+    mkString(*vLocks, lockedFlake.lockFile.to_string());
+
+    emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
+
+    mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
+
+    static RootValue vCallFlake = nullptr;
+
+    if (!vCallFlake) {
+        vCallFlake = allocRootValue(state.allocValue());
+        state.eval(state.parseExprFromString(
+            #include "call-flake.nix.gen.hh"
+            , "/"), **vCallFlake);
+    }
+
+    state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
+    state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
+}
+
+static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
+    auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
+    if (evalSettings.pureEval && !flakeRef.input.isImmutable())
+        throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
+
+    callFlake(state,
+        lockFlake(state, flakeRef,
+            LockFlags {
+                .updateLockFile = false,
+                .useRegistries = !evalSettings.pureEval,
+                .allowMutable  = !evalSettings.pureEval,
+            }),
+        v);
+}
+
+static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes");
+
+}
+
+Fingerprint LockedFlake::getFingerprint() const
+{
+    // FIXME: as an optimization, if the flake contains a lock file
+    // and we haven't changed it, then it's sufficient to use
+    // flake.sourceInfo.storePath for the fingerprint.
+    return hashString(htSHA256,
+        fmt("%s;%d;%d;%s",
+            flake.sourceInfo->storePath.to_string(),
+            flake.lockedRef.input.getRevCount().value_or(0),
+            flake.lockedRef.input.getLastModified().value_or(0),
+            lockFile));
+}
+
+Flake::~Flake() { }
+
+}

src/libexpr/flake/flake.hh

@@ -0,0 +1,110 @@
+#pragma once
+
+#include "types.hh"
+#include "flakeref.hh"
+#include "lockfile.hh"
+#include "value.hh"
+
+namespace nix {
+
+class EvalState;
+
+namespace fetchers { struct Tree; }
+
+namespace flake {
+
+struct FlakeInput;
+
+typedef std::map<FlakeId, FlakeInput> FlakeInputs;
+
+struct FlakeInput
+{
+    std::optional<FlakeRef> ref;
+    bool isFlake = true;
+    std::optional<InputPath> follows;
+    bool absolute = false; // whether 'follows' is relative to the flake root
+    FlakeInputs overrides;
+};
+
+struct Flake
+{
+    FlakeRef originalRef;
+    FlakeRef resolvedRef;
+    FlakeRef lockedRef;
+    std::optional<std::string> description;
+    std::shared_ptr<const fetchers::Tree> sourceInfo;
+    FlakeInputs inputs;
+    ~Flake();
+};
+
+Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
+
+/* Fingerprint of a locked flake; used as a cache key. */
+typedef Hash Fingerprint;
+
+struct LockedFlake
+{
+    Flake flake;
+    LockFile lockFile;
+
+    Fingerprint getFingerprint() const;
+};
+
+struct LockFlags
+{
+    /* Whether to ignore the existing lock file, creating a new one
+       from scratch. */
+    bool recreateLockFile = false;
+
+    /* Whether to update the lock file at all. If set to false, if any
+       change to the lock file is needed (e.g. when an input has been
+       added to flake.nix), you get a fatal error. */
+    bool updateLockFile = true;
+
+    /* Whether to write the lock file to disk. If set to true, if the
+       any changes to the lock file are needed and the flake is not
+       writable (i.e. is not a local Git working tree or similar), you
+       get a fatal error. If set to false, Nix will use the modified
+       lock file in memory only, without writing it to disk. */
+    bool writeLockFile = true;
+
+    /* Whether to use the registries to lookup indirect flake
+       references like 'nixpkgs'. */
+    bool useRegistries = true;
+
+    /* Whether mutable flake references (i.e. those without a Git
+       revision or similar) without a corresponding lock are
+       allowed. Mutable flake references with a lock are always
+       allowed. */
+    bool allowMutable = true;
+
+    /* Whether to commit changes to flake.lock. */
+    bool commitLockFile = false;
+
+    /* Flake inputs to be overriden. */
+    std::map<InputPath, FlakeRef> inputOverrides;
+
+    /* Flake inputs to be updated. This means that any existing lock
+       for those inputs will be ignored. */
+    std::set<InputPath> inputUpdates;
+};
+
+LockedFlake lockFlake(
+    EvalState & state,
+    const FlakeRef & flakeRef,
+    const LockFlags & lockFlags);
+
+void callFlake(
+    EvalState & state,
+    const LockedFlake & lockedFlake,
+    Value & v);
+
+}
+
+void emitTreeAttrs(
+    EvalState & state,
+    const fetchers::Tree & tree,
+    const fetchers::Input & input,
+    Value & v, bool emptyRevFallback = false);
+
+}

src/libexpr/flake/flakeref.cc

@@ -0,0 +1,206 @@
+#include "flakeref.hh"
+#include "store-api.hh"
+#include "url.hh"
+#include "url-parts.hh"
+#include "fetchers.hh"
+#include "registry.hh"
+
+namespace nix {
+
+#if 0
+// 'dir' path elements cannot start with a '.'. We also reject
+// potentially dangerous characters like ';'.
+const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
+const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
+#endif
+
+std::string FlakeRef::to_string() const
+{
+    std::map<std::string, std::string> extraQuery;
+    if (subdir != "")
+        extraQuery.insert_or_assign("dir", subdir);
+    return input.toURLString(extraQuery);
+}
+
+fetchers::Attrs FlakeRef::toAttrs() const
+{
+    auto attrs = input.toAttrs();
+    if (subdir != "")
+        attrs.emplace("dir", subdir);
+    return attrs;
+}
+
+std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
+{
+    str << flakeRef.to_string();
+    return str;
+}
+
+bool FlakeRef::operator ==(const FlakeRef & other) const
+{
+    return input == other.input && subdir == other.subdir;
+}
+
+FlakeRef FlakeRef::resolve(ref<Store> store) const
+{
+    auto [input2, extraAttrs] = lookupInRegistries(store, input);
+    return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
+}
+
+FlakeRef parseFlakeRef(
+    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+{
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
+    if (fragment != "")
+        throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
+    return flakeRef;
+}
+
+std::optional<FlakeRef> maybeParseFlakeRef(
+    const std::string & url, const std::optional<Path> & baseDir)
+{
+    try {
+        return parseFlakeRef(url, baseDir);
+    } catch (Error &) {
+        return {};
+    }
+}
+
+std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+{
+    using namespace fetchers;
+
+    static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+";
+
+    static std::regex pathUrlRegex(
+        "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)"
+        + "(?:\\?(" + queryRegex + "))?"
+        + "(?:#(" + queryRegex + "))?",
+        std::regex::ECMAScript);
+
+    static std::regex flakeRegex(
+        "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
+        + "(?:#(" + queryRegex + "))?",
+        std::regex::ECMAScript);
+
+    std::smatch match;
+
+    /* Check if 'url' is a flake ID. This is an abbreviated syntax for
+       'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
+
+    if (std::regex_match(url, match, flakeRegex)) {
+        auto parsedURL = ParsedURL{
+            .url = url,
+            .base = "flake:" + std::string(match[1]),
+            .scheme = "flake",
+            .authority = "",
+            .path = match[1],
+        };
+
+        return std::make_pair(
+            FlakeRef(Input::fromURL(parsedURL), ""),
+            percentDecode(std::string(match[6])));
+    }
+
+    else if (std::regex_match(url, match, pathUrlRegex)) {
+        std::string path = match[1];
+        std::string fragment = percentDecode(std::string(match[3]));
+
+        if (baseDir) {
+            /* Check if 'url' is a path (either absolute or relative
+               to 'baseDir'). If so, search upward to the root of the
+               repo (i.e. the directory containing .git). */
+
+            path = absPath(path, baseDir, true);
+
+            if (!S_ISDIR(lstat(path).st_mode))
+                throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+
+            if (!allowMissing && !pathExists(path + "/flake.nix"))
+                throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+
+            auto flakeRoot = path;
+            std::string subdir;
+
+            while (flakeRoot != "/") {
+                if (pathExists(flakeRoot + "/.git")) {
+                    auto base = std::string("git+file://") + flakeRoot;
+
+                    auto parsedURL = ParsedURL{
+                        .url = base, // FIXME
+                        .base = base,
+                        .scheme = "git+file",
+                        .authority = "",
+                        .path = flakeRoot,
+                        .query = decodeQuery(match[2]),
+                    };
+
+                    if (subdir != "") {
+                        if (parsedURL.query.count("dir"))
+                            throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+                        parsedURL.query.insert_or_assign("dir", subdir);
+                    }
+
+                    if (pathExists(flakeRoot + "/.git/shallow"))
+                        parsedURL.query.insert_or_assign("shallow", "1");
+
+                    return std::make_pair(
+                        FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+                        fragment);
+                }
+
+                subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
+                flakeRoot = dirOf(flakeRoot);
+            }
+
+        } else {
+            if (!hasPrefix(path, "/"))
+                throw BadURL("flake reference '%s' is not an absolute path", url);
+            auto query = decodeQuery(match[2]);
+            path = canonPath(path + "/" + get(query, "dir").value_or(""));
+        }
+
+        fetchers::Attrs attrs;
+        attrs.insert_or_assign("type", "path");
+        attrs.insert_or_assign("path", path);
+
+        return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment);
+    }
+
+    else {
+        auto parsedURL = parseURL(url);
+        std::string fragment;
+        std::swap(fragment, parsedURL.fragment);
+        return std::make_pair(
+            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+            fragment);
+    }
+}
+
+std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir)
+{
+    try {
+        return parseFlakeRefWithFragment(url, baseDir);
+    } catch (Error & e) {
+        return {};
+    }
+}
+
+FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
+{
+    auto attrs2(attrs);
+    attrs2.erase("dir");
+    return FlakeRef(
+        fetchers::Input::fromAttrs(std::move(attrs2)),
+        fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
+}
+
+std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
+{
+    auto [tree, lockedInput] = input.fetch(store);
+    return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
+}
+
+}

src/libexpr/flake/flakeref.hh

@@ -0,0 +1,53 @@
+#pragma once
+
+#include "types.hh"
+#include "hash.hh"
+#include "fetchers.hh"
+
+#include <variant>
+
+namespace nix {
+
+class Store;
+
+typedef std::string FlakeId;
+
+struct FlakeRef
+{
+    fetchers::Input input;
+
+    Path subdir;
+
+    bool operator==(const FlakeRef & other) const;
+
+    FlakeRef(fetchers::Input && input, const Path & subdir)
+        : input(std::move(input)), subdir(subdir)
+    { }
+
+    // FIXME: change to operator <<.
+    std::string to_string() const;
+
+    fetchers::Attrs toAttrs() const;
+
+    FlakeRef resolve(ref<Store> store) const;
+
+    static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
+
+    std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const;
+};
+
+std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
+
+FlakeRef parseFlakeRef(
+    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+
+std::optional<FlakeRef> maybeParseFlake(
+    const std::string & url, const std::optional<Path> & baseDir = {});
+
+std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+
+std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir = {});
+
+}

src/libexpr/flake/lockfile.cc

@@ -0,0 +1,339 @@
+#include "lockfile.hh"
+#include "store-api.hh"
+#include "url-parts.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::flake {
+
+FlakeRef getFlakeRef(
+    const nlohmann::json & json,
+    const char * attr,
+    const char * info)
+{
+    auto i = json.find(attr);
+    if (i != json.end()) {
+        auto attrs = fetchers::jsonToAttrs(*i);
+        // FIXME: remove when we drop support for version 5.
+        if (info) {
+            auto j = json.find(info);
+            if (j != json.end()) {
+                for (auto k : fetchers::jsonToAttrs(*j))
+                    attrs.insert_or_assign(k.first, k.second);
+            }
+        }
+        return FlakeRef::fromAttrs(attrs);
+    }
+
+    throw Error("attribute '%s' missing in lock file", attr);
+}
+
+LockedNode::LockedNode(const nlohmann::json & json)
+    : lockedRef(getFlakeRef(json, "locked", "info"))
+    , originalRef(getFlakeRef(json, "original", nullptr))
+    , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
+{
+    if (!lockedRef.input.isImmutable())
+        throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
+}
+
+StorePath LockedNode::computeStorePath(Store & store) const
+{
+    return lockedRef.input.computeStorePath(store);
+}
+
+std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
+{
+    auto pos = root;
+
+    if (!pos) return {};
+
+    for (auto & elem : path) {
+        if (auto i = get(pos->inputs, elem)) {
+            if (auto node = std::get_if<0>(&*i))
+                pos = *node;
+            else if (auto follows = std::get_if<1>(&*i)) {
+                pos = findInput(*follows);
+                if (!pos) return {};
+            }
+        } else
+            return {};
+    }
+
+    return pos;
+}
+
+LockFile::LockFile(const nlohmann::json & json, const Path & path)
+{
+    auto version = json.value("version", 0);
+    if (version < 5 || version > 7)
+        throw Error("lock file '%s' has unsupported version %d", path, version);
+
+    std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap;
+
+    std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs;
+
+    getInputs = [&](Node & node, const nlohmann::json & jsonNode)
+    {
+        if (jsonNode.find("inputs") == jsonNode.end()) return;
+        for (auto & i : jsonNode["inputs"].items()) {
+            if (i.value().is_array()) {
+                InputPath path;
+                for (auto & j : i.value())
+                    path.push_back(j);
+                node.inputs.insert_or_assign(i.key(), path);
+            } else {
+                std::string inputKey = i.value();
+                auto k = nodeMap.find(inputKey);
+                if (k == nodeMap.end()) {
+                    auto jsonNode2 = json["nodes"][inputKey];
+                    auto input = std::make_shared<LockedNode>(jsonNode2);
+                    k = nodeMap.insert_or_assign(inputKey, input).first;
+                    getInputs(*input, jsonNode2);
+                }
+                if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second))
+                    node.inputs.insert_or_assign(i.key(), child);
+                else
+                    // FIXME: replace by follows node
+                    throw Error("lock file contains cycle to root node");
+            }
+        }
+    };
+
+    std::string rootKey = json["root"];
+    nodeMap.insert_or_assign(rootKey, root);
+    getInputs(*root, json["nodes"][rootKey]);
+
+    // FIXME: check that there are no cycles in version >= 7. Cycles
+    // between inputs are only possible using 'follows' indirections.
+    // Once we drop support for version <= 6, we can simplify the code
+    // a bit since we don't need to worry about cycles.
+}
+
+nlohmann::json LockFile::toJson() const
+{
+    nlohmann::json nodes;
+    std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
+    std::unordered_set<std::string> keys;
+
+    std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode;
+
+    dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string
+    {
+        auto k = nodeKeys.find(node);
+        if (k != nodeKeys.end())
+            return k->second;
+
+        if (!keys.insert(key).second) {
+            for (int n = 2; ; ++n) {
+                auto k = fmt("%s_%d", key, n);
+                if (keys.insert(k).second) {
+                    key = k;
+                    break;
+                }
+            }
+        }
+
+        nodeKeys.insert_or_assign(node, key);
+
+        auto n = nlohmann::json::object();
+
+        if (!node->inputs.empty()) {
+            auto inputs = nlohmann::json::object();
+            for (auto & i : node->inputs) {
+                if (auto child = std::get_if<0>(&i.second)) {
+                    inputs[i.first] = dumpNode(i.first, *child);
+                } else if (auto follows = std::get_if<1>(&i.second)) {
+                    auto arr = nlohmann::json::array();
+                    for (auto & x : *follows)
+                        arr.push_back(x);
+                    inputs[i.first] = std::move(arr);
+                }
+            }
+            n["inputs"] = std::move(inputs);
+        }
+
+        if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
+            n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
+            n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
+            if (!lockedNode->isFlake) n["flake"] = false;
+        }
+
+        nodes[key] = std::move(n);
+
+        return key;
+    };
+
+    nlohmann::json json;
+    json["version"] = 7;
+    json["root"] = dumpNode("root", root);
+    json["nodes"] = std::move(nodes);
+
+    return json;
+}
+
+std::string LockFile::to_string() const
+{
+    return toJson().dump(2);
+}
+
+LockFile LockFile::read(const Path & path)
+{
+    if (!pathExists(path)) return LockFile();
+    return LockFile(nlohmann::json::parse(readFile(path)), path);
+}
+
+std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
+{
+    stream << lockFile.toJson().dump(2);
+    return stream;
+}
+
+void LockFile::write(const Path & path) const
+{
+    createDirs(dirOf(path));
+    writeFile(path, fmt("%s\n", *this));
+}
+
+bool LockFile::isImmutable() const
+{
+    std::unordered_set<std::shared_ptr<const Node>> nodes;
+
+    std::function<void(std::shared_ptr<const Node> node)> visit;
+
+    visit = [&](std::shared_ptr<const Node> node)
+    {
+        if (!nodes.insert(node).second) return;
+        for (auto & i : node->inputs)
+            if (auto child = std::get_if<0>(&i.second))
+                visit(*child);
+    };
+
+    visit(root);
+
+    for (auto & i : nodes) {
+        if (i == root) continue;
+        auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
+        if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false;
+    }
+
+    return true;
+}
+
+bool LockFile::operator ==(const LockFile & other) const
+{
+    // FIXME: slow
+    return toJson() == other.toJson();
+}
+
+InputPath parseInputPath(std::string_view s)
+{
+    InputPath path;
+
+    for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
+        if (!std::regex_match(elem, flakeIdRegex))
+            throw UsageError("invalid flake input path element '%s'", elem);
+        path.push_back(elem);
+    }
+
+    return path;
+}
+
+std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
+{
+    std::unordered_set<std::shared_ptr<Node>> done;
+    std::map<InputPath, Node::Edge> res;
+
+    std::function<void(const InputPath & prefix, std::shared_ptr<Node> node)> recurse;
+
+    recurse = [&](const InputPath & prefix, std::shared_ptr<Node> node)
+    {
+        if (!done.insert(node).second) return;
+
+        for (auto &[id, input] : node->inputs) {
+            auto inputPath(prefix);
+            inputPath.push_back(id);
+            res.emplace(inputPath, input);
+            if (auto child = std::get_if<0>(&input))
+                recurse(inputPath, *child);
+        }
+    };
+
+    recurse({}, root);
+
+    return res;
+}
+
+std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
+{
+    if (auto node = std::get_if<0>(&edge))
+        stream << "'" << (*node)->lockedRef << "'";
+    else if (auto follows = std::get_if<1>(&edge))
+        stream << fmt("follows '%s'", printInputPath(*follows));
+    return stream;
+}
+
+static bool equals(const Node::Edge & e1, const Node::Edge & e2)
+{
+    if (auto n1 = std::get_if<0>(&e1))
+        if (auto n2 = std::get_if<0>(&e2))
+            return (*n1)->lockedRef == (*n2)->lockedRef;
+    if (auto f1 = std::get_if<1>(&e1))
+        if (auto f2 = std::get_if<1>(&e2))
+            return *f1 == *f2;
+    return false;
+}
+
+std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
+{
+    auto oldFlat = oldLocks.getAllInputs();
+    auto newFlat = newLocks.getAllInputs();
+
+    auto i = oldFlat.begin();
+    auto j = newFlat.begin();
+    std::string res;
+
+    while (i != oldFlat.end() || j != newFlat.end()) {
+        if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
+            res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second);
+            ++j;
+        } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
+            res += fmt("* Removed '%s'\n", printInputPath(i->first));
+            ++i;
+        } else {
+            if (!equals(i->second, j->second)) {
+                res += fmt("* Updated '%s': %s -> %s\n",
+                    printInputPath(i->first),
+                    i->second,
+                    j->second);
+            }
+            ++i;
+            ++j;
+        }
+    }
+
+    return res;
+}
+
+void LockFile::check()
+{
+    auto inputs = getAllInputs();
+
+    for (auto & [inputPath, input] : inputs) {
+        if (auto follows = std::get_if<1>(&input)) {
+            if (!follows->empty() && !get(inputs, *follows))
+                throw Error("input '%s' follows a non-existent input '%s'",
+                    printInputPath(inputPath),
+                    printInputPath(*follows));
+        }
+    }
+}
+
+void check();
+
+std::string printInputPath(const InputPath & path)
+{
+    return concatStringsSep("/", path);
+}
+
+}

src/libexpr/flake/lockfile.hh

@@ -0,0 +1,83 @@
+#pragma once
+
+#include "flakeref.hh"
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+class Store;
+class StorePath;
+}
+
+namespace nix::flake {
+
+typedef std::vector<FlakeId> InputPath;
+
+struct LockedNode;
+
+/* A node in the lock file. It has outgoing edges to other nodes (its
+   inputs). Only the root node has this type; all other nodes have
+   type LockedNode. */
+struct Node : std::enable_shared_from_this<Node>
+{
+    typedef std::variant<std::shared_ptr<LockedNode>, InputPath> Edge;
+
+    std::map<FlakeId, Edge> inputs;
+
+    virtual ~Node() { }
+};
+
+/* A non-root node in the lock file. */
+struct LockedNode : Node
+{
+    FlakeRef lockedRef, originalRef;
+    bool isFlake = true;
+
+    LockedNode(
+        const FlakeRef & lockedRef,
+        const FlakeRef & originalRef,
+        bool isFlake = true)
+        : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake)
+    { }
+
+    LockedNode(const nlohmann::json & json);
+
+    StorePath computeStorePath(Store & store) const;
+};
+
+struct LockFile
+{
+    std::shared_ptr<Node> root = std::make_shared<Node>();
+
+    LockFile() {};
+    LockFile(const nlohmann::json & json, const Path & path);
+
+    nlohmann::json toJson() const;
+
+    std::string to_string() const;
+
+    static LockFile read(const Path & path);
+
+    void write(const Path & path) const;
+
+    bool isImmutable() const;
+
+    bool operator ==(const LockFile & other) const;
+
+    std::shared_ptr<Node> findInput(const InputPath & path);
+
+    std::map<InputPath, Node::Edge> getAllInputs() const;
+
+    static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
+
+    /* Check that every 'follows' input target exists. */
+    void check();
+};
+
+std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
+
+InputPath parseInputPath(std::string_view s);
+
+std::string printInputPath(const InputPath & path);
+
+}

src/libexpr/get-drvs.cc

@@ -38,8 +38,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
     auto i = drv.outputs.find(outputName);
     if (i == drv.outputs.end())
         throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
+    auto & [outputName, output] = *i;
 
-    outPath = store->printStorePath(i->second.path);
+    auto optStorePath = output.path(*store, drv.name, outputName);
+    if (optStorePath)
+        outPath = store->printStorePath(*optStorePath);
 }
 
 
@@ -77,12 +80,15 @@ string DrvInfo::queryDrvPath() const
 
 string DrvInfo::queryOutPath() const
 {
-    if (outPath == "" && attrs) {
+    if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
         PathSet context;
-        outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
+        if (i != attrs->end())
+            outPath = state->coerceToPath(*i->pos, *i->value, context);
     }
-    return outPath;
+    if (!outPath)
+        throw UnimplementedError("CA derivations are not yet supported");
+    return *outPath;
 }
 
 

src/libexpr/get-drvs.hh

@@ -20,7 +20,7 @@ private:
     mutable string name;
     mutable string system;
     mutable string drvPath;
-    mutable string outPath;
+    mutable std::optional<string> outPath;
     mutable string outputName;
     Outputs outputs;
 

src/libexpr/json-to-value.cc

@@ -115,6 +115,14 @@ public:
     {
         return handle_value<void(Value&, const char*)>(mkString, val.c_str());
     }
+#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
+    bool binary(binary_t&)
+    {
+        // This function ought to be unreachable
+        assert(false);
+        return true;
+    }
+#endif
 
     bool start_object(std::size_t len)
     {

src/libexpr/local.mk

@@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr
 
 libexpr_DIR := $(d)
 
-libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
+libexpr_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+  $(wildcard $(d)/primops/*.cc) \
+  $(wildcard $(d)/flake/*.cc) \
+  $(d)/lexer-tab.cc \
+  $(d)/parser-tab.cc
 
 libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
 
@@ -34,4 +39,9 @@ dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
 
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
+$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
+
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh
+
+$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh

src/libexpr/parser.y

@@ -614,8 +614,7 @@ Path resolveExprPath(Path path)
         // Basic cycle/depth limit to avoid infinite loops.
         if (++followCount >= maxFollow)
             throw Error("too many symbolic links encountered while traversing the path '%s'", path);
-        if (lstat(path.c_str(), &st))
-            throw SysError("getting status of '%s'", path);
+        st = lstat(path);
         if (!S_ISLNK(st.st_mode)) break;
         path = absPath(readLink(path), dirOf(path));
     }
@@ -719,7 +718,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
     if (isUri(elem.second)) {
         try {
             res = { true, store->toRealPath(fetchers::downloadTarball(
-                        store, resolveUri(elem.second), "source", false).storePath) };
+                        store, resolveUri(elem.second), "source", false).first.storePath) };
         } catch (FileTransferError & e) {
             logWarning({
                 .name = "Entry download",

src/libexpr/primops.hh

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "eval.hh"
 
 #include <tuple>
@@ -10,9 +12,11 @@ struct RegisterPrimOp
     struct Info
     {
         std::string name;
-        size_t arity;
-        PrimOpFun primOp;
+        std::vector<std::string> args;
+        size_t arity = 0;
+        const char * doc;
         std::optional<std::string> requiredFeature;
+        PrimOpFun fun;
     };
 
     typedef std::vector<Info> PrimOps;
@@ -26,6 +30,8 @@ struct RegisterPrimOp
         size_t arity,
         PrimOpFun fun,
         std::optional<std::string> requiredFeature = {});
+
+    RegisterPrimOp(Info && info);
 };
 
 /* These primops are disabled without enableNativeCode, but plugins

src/libexpr/primops/context.cc

@@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos,
     mkString(v, s, PathSet());
 }
 
-static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
+static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
 
 
 static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -21,7 +21,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args,
     mkBool(v, !context.empty());
 }
 
-static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
+static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext);
 
 
 /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
@@ -42,7 +42,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po
     mkString(v, s, context2);
 }
 
-static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
 
 
 /* Extract the context of a string as a structured Nix value.
@@ -127,7 +127,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
     v.attrs->sort();
 }
 
-static RegisterPrimOp r4("__getContext", 1, prim_getContext);
+static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext);
 
 
 /* Append the given context to a given string.
@@ -191,6 +191,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
     mkString(v, orig, context);
 }
 
-static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
+static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext);
 
 }

src/libexpr/primops/derivation.nix

@@ -0,0 +1,27 @@
+/* This is the implementation of the ‘derivation’ builtin function.
+   It's actually a wrapper around the ‘derivationStrict’ primop. */
+
+drvAttrs @ { outputs ? [ "out" ], ... }:
+
+let
+
+  strict = derivationStrict drvAttrs;
+
+  commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) //
+    { all = map (x: x.value) outputsList;
+      inherit drvAttrs;
+    };
+
+  outputToAttrListElement = outputName:
+    { name = outputName;
+      value = commonAttrs // {
+        outPath = builtins.getAttr outputName strict;
+        drvPath = strict.drvPath;
+        type = "derivation";
+        inherit outputName;
+      };
+    };
+
+  outputsList = map outputToAttrListElement outputs;
+
+in (builtins.head outputsList).value

src/libexpr/primops/fetchGit.cc

@@ -1,91 +0,0 @@
-#include "primops.hh"
-#include "eval-inline.hh"
-#include "store-api.hh"
-#include "hash.hh"
-#include "fetchers.hh"
-#include "url.hh"
-
-namespace nix {
-
-static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    std::string url;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
-    std::string name = "source";
-    bool fetchSubmodules = false;
-    PathSet context;
-
-    state.forceValue(*args[0]);
-
-    if (args[0]->type == tAttrs) {
-
-        state.forceAttrs(*args[0], pos);
-
-        for (auto & attr : *args[0]->attrs) {
-            string n(attr.name);
-            if (n == "url")
-                url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
-            else if (n == "ref")
-                ref = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "rev")
-                rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
-            else if (n == "name")
-                name = state.forceStringNoCtx(*attr.value, *attr.pos);
-            else if (n == "submodules")
-                fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
-            else
-                throw EvalError({
-                    .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
-                    .errPos = *attr.pos
-                });
-        }
-
-        if (url.empty())
-            throw EvalError({
-                .hint = hintfmt("'url' argument required"),
-                .errPos = pos
-            });
-
-    } else
-        url = state.coerceToString(pos, *args[0], context, false, false);
-
-    // FIXME: git externals probably can be used to bypass the URI
-    // whitelist. Ah well.
-    state.checkURI(url);
-
-    if (evalSettings.pureEval && !rev)
-        throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
-    fetchers::Attrs attrs;
-    attrs.insert_or_assign("type", "git");
-    attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
-    if (ref) attrs.insert_or_assign("ref", *ref);
-    if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
-    auto input = fetchers::inputFromAttrs(attrs);
-
-    // FIXME: use name?
-    auto [tree, input2] = input->fetchTree(state.store);
-
-    state.mkAttrs(v, 8);
-    auto storePath = state.store->printStorePath(tree.storePath);
-    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
-    // Backward compatibility: set 'rev' to
-    // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
-    // Backward compatibility: set 'revCount' to 0 for a dirty tree.
-    mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
-        tree.info.revCount.value_or(0));
-    mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
-    v.attrs->sort();
-
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
-}
-
-static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
-
-}

src/libexpr/primops/fetchMercurial.cc

@@ -3,8 +3,7 @@
 #include "store-api.hh"
 #include "fetchers.hh"
 #include "url.hh"
-
-#include <regex>
+#include "url-parts.hh"
 
 namespace nix {
 
@@ -31,7 +30,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
                 // be both a revision or a branch/tag name.
                 auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
                 if (std::regex_match(value, revRegex))
-                    rev = Hash(value, htSHA1);
+                    rev = Hash::parseAny(value, htSHA1);
                 else
                     ref = value;
             }
@@ -65,29 +64,29 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
-    auto input = fetchers::inputFromAttrs(attrs);
+    auto input = fetchers::Input::fromAttrs(std::move(attrs));
 
     // FIXME: use name
-    auto [tree, input2] = input->fetchTree(state.store);
+    auto [tree, input2] = input.fetch(state.store);
 
     state.mkAttrs(v, 8);
     auto storePath = state.store->printStorePath(tree.storePath);
     mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
-    if (input2->getRef())
-        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
+    if (input2.getRef())
+        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef());
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+    auto rev2 = input2.getRev().value_or(Hash(htSHA1));
     mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
     mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
-    if (tree.info.revCount)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+    if (auto revCount = input2.getRevCount())
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
     v.attrs->sort();
 
     if (state.allowedPaths)
         state.allowedPaths->insert(tree.actualPath);
 }
 
-static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
+static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);
 
 }

src/libexpr/primops/fetchTree.cc

@@ -3,6 +3,7 @@
 #include "store-api.hh"
 #include "fetchers.hh"
 #include "filetransfer.hh"
+#include "registry.hh"
 
 #include <ctime>
 #include <iomanip>
@@ -12,39 +13,73 @@ namespace nix {
 void emitTreeAttrs(
     EvalState & state,
     const fetchers::Tree & tree,
-    std::shared_ptr<const fetchers::Input> input,
-    Value & v)
+    const fetchers::Input & input,
+    Value & v,
+    bool emptyRevFallback)
 {
+    assert(input.isImmutable());
+
     state.mkAttrs(v, 8);
 
     auto storePath = state.store->printStorePath(tree.storePath);
 
     mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
 
-    assert(tree.info.narHash);
-    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
-        tree.info.narHash.to_string(SRI, true));
+    // FIXME: support arbitrary input attributes.
 
-    if (input->getRev()) {
-        mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
-        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
+    auto narHash = input.getNarHash();
+    assert(narHash);
+    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
+        narHash->to_string(SRI, true));
+
+    if (auto rev = input.getRev()) {
+        mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
+        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
+    } else if (emptyRevFallback) {
+        // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
+        auto emptyHash = Hash(htSHA1);
+        mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
+        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
     }
 
-    if (tree.info.revCount)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+    if (input.getType() == "git")
+        mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
 
-    if (tree.info.lastModified)
-        mkString(*state.allocAttr(v, state.symbols.create("lastModified")),
-            fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
+    if (auto revCount = input.getRevCount())
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
+    else if (emptyRevFallback)
+        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
+
+    if (auto lastModified = input.getLastModified()) {
+        mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
+        mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
+            fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
+    }
 
     v.attrs->sort();
 }
 
-static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+std::string fixURI(std::string uri, EvalState &state)
 {
-    settings.requireExperimentalFeature("flakes");
+    state.checkURI(uri);
+    return uri.find("://") != std::string::npos ? uri : "file://" + uri;
+}
 
-    std::shared_ptr<const fetchers::Input> input;
+void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
+{
+    string n(name);
+    attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
+}
+
+static void fetchTree(
+    EvalState &state,
+    const Pos &pos,
+    Value **args,
+    Value &v,
+    const std::optional<std::string> type,
+    bool emptyRevFallback = false
+) {
+    fetchers::Input input;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -56,38 +91,68 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
 
         for (auto & attr : *args[0]->attrs) {
             state.forceValue(*attr.value);
-            if (attr.value->type == tString)
-                attrs.emplace(attr.name, attr.value->string.s);
+            if (attr.value->type == tPath || attr.value->type == tString)
+                addURI(
+                    state,
+                    attrs,
+                    attr.name,
+                    state.coerceToString(*attr.pos, *attr.value, context, false, false)
+                );
+            else if (attr.value->type == tString)
+                addURI(state, attrs, attr.name, attr.value->string.s);
             else if (attr.value->type == tBool)
-                attrs.emplace(attr.name, attr.value->boolean);
+                attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
+            else if (attr.value->type == tInt)
+                attrs.emplace(attr.name, attr.value->integer);
             else
-                throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected",
+                throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
                     attr.name, showType(*attr.value));
         }
 
+        if (type)
+            attrs.emplace("type", type.value());
+
         if (!attrs.count("type"))
             throw Error({
                 .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
                 .errPos = pos
             });
 
-        input = fetchers::inputFromAttrs(attrs);
-    } else
-        input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
+        input = fetchers::Input::fromAttrs(std::move(attrs));
+    } else {
+        auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
 
-    if (evalSettings.pureEval && !input->isImmutable())
-        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input");
+        if (type == "git") {
+            fetchers::Attrs attrs;
+            attrs.emplace("type", "git");
+            attrs.emplace("url", url);
+            input = fetchers::Input::fromAttrs(std::move(attrs));
+        } else {
+            input = fetchers::Input::fromURL(url);
+        }
+    }
 
-    // FIXME: use fetchOrSubstituteTree
-    auto [tree, input2] = input->fetchTree(state.store);
+    if (!evalSettings.pureEval && !input.isDirect())
+        input = lookupInRegistries(state.store, input).first;
+
+    if (evalSettings.pureEval && !input.isImmutable())
+        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);
+
+    auto [tree, input2] = input.fetch(state.store);
 
     if (state.allowedPaths)
         state.allowedPaths->insert(tree.actualPath);
 
-    emitTreeAttrs(state, tree, input2, v);
+    emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
 }
 
-static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
+static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    settings.requireExperimentalFeature("flakes");
+    fetchTree(state, pos, args, v, std::nullopt);
+}
+
+static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree);
 
 static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     const string & who, bool unpack, std::string name)
@@ -136,7 +201,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     auto storePath =
         unpack
-        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
+        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
         : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
     auto path = state.store->toRealPath(storePath);
@@ -161,12 +226,187 @@ static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Va
     fetch(state, pos, args, v, "fetchurl", false, "");
 }
 
+static RegisterPrimOp primop_fetchurl({
+    .name = "__fetchurl",
+    .args = {"url"},
+    .doc = R"(
+      Download the specified URL and return the path of the downloaded
+      file. This function is not available if [restricted evaluation
+      mode](../command-ref/conf-file.md) is enabled.
+    )",
+    .fun = prim_fetchurl,
+});
+
 static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     fetch(state, pos, args, v, "fetchTarball", true, "source");
 }
 
-static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
-static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+static RegisterPrimOp primop_fetchTarball({
+    .name = "fetchTarball",
+    .args = {"args"},
+    .doc = R"(
+      Download the specified URL, unpack it and return the path of the
+      unpacked tree. The file must be a tape archive (`.tar`) compressed
+      with `gzip`, `bzip2` or `xz`. The top-level path component of the
+      files in the tarball is removed, so it is best if the tarball
+      contains a single directory at top level. The typical use of the
+      function is to obtain external Nix expression dependencies, such as
+      a particular version of Nixpkgs, e.g.
+
+      ```nix
+      with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
+
+      stdenv.mkDerivation { … }
+      ```
+
+      The fetched tarball is cached for a certain amount of time (1 hour
+      by default) in `~/.cache/nix/tarballs/`. You can change the cache
+      timeout either on the command line with `--option tarball-ttl number
+      of seconds` or in the Nix configuration file with this option: ` 
+      number of seconds to cache `.
+
+      Note that when obtaining the hash with ` nix-prefetch-url ` the
+      option `--unpack` is required.
+
+      This function can also verify the contents against a hash. In that
+      case, the function takes a set instead of a URL. The set requires
+      the attribute `url` and the attribute `sha256`, e.g.
+
+      ```nix
+      with import (fetchTarball {
+        url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
+        sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
+      }) {};
+
+      stdenv.mkDerivation { … }
+      ```
+
+      This function is not available if [restricted evaluation
+      mode](../command-ref/conf-file.md) is enabled.
+    )",
+    .fun = prim_fetchTarball,
+});
+
+static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+{
+    fetchTree(state, pos, args, v, "git", true);
+}
+
+static RegisterPrimOp primop_fetchGit({
+    .name = "fetchGit",
+    .args = {"args"},
+    .doc = R"(
+      Fetch a path from git. *args* can be a URL, in which case the HEAD
+      of the repo at that URL is fetched. Otherwise, it can be an
+      attribute with the following attributes (all except `url` optional):
+
+        - url  
+          The URL of the repo.
+
+        - name  
+          The name of the directory the repo should be exported to in the
+          store. Defaults to the basename of the URL.
+
+        - rev  
+          The git revision to fetch. Defaults to the tip of `ref`.
+
+        - ref  
+          The git ref to look for the requested revision under. This is
+          often a branch or tag name. Defaults to `HEAD`.
+
+          By default, the `ref` value is prefixed with `refs/heads/`. As
+          of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts
+          with `refs/`.
+
+        - submodules  
+          A Boolean parameter that specifies whether submodules should be
+          checked out. Defaults to `false`.
+
+      Here are some examples of how to use `fetchGit`.
+
+        - To fetch a private repository over SSH:
+
+          ```nix
+          builtins.fetchGit {
+            url = "git@github.com:my-secret/repository.git";
+            ref = "master";
+            rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
+          }
+          ```
+
+        - To fetch an arbitrary reference:
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/NixOS/nix.git";
+            ref = "refs/heads/0.5-release";
+          }
+          ```
+
+        - If the revision you're looking for is in the default branch of
+          the git repository you don't strictly need to specify the branch
+          name in the `ref` attribute.
+
+          However, if the revision you're looking for is in a future
+          branch for the non-default branch you will need to specify the
+          the `ref` attribute as well.
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/nixos/nix.git";
+            rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+            ref = "1.11-maintenance";
+          }
+          ```
+
+          > **Note**
+          > 
+          > It is nice to always specify the branch which a revision
+          > belongs to. Without the branch being specified, the fetcher
+          > might fail if the default branch changes. Additionally, it can
+          > be confusing to try a commit from a non-default branch and see
+          > the fetch fail. If the branch is specified the fault is much
+          > more obvious.
+
+        - If the revision you're looking for is in the default branch of
+          the git repository you may omit the `ref` attribute.
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/nixos/nix.git";
+            rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
+          }
+          ```
+
+        - To fetch a specific tag:
+
+          ```nix
+          builtins.fetchGit {
+            url = "https://github.com/nixos/nix.git";
+            ref = "refs/tags/1.9";
+          }
+          ```
+
+        - To fetch the latest version of a remote branch:
+
+          ```nix
+          builtins.fetchGit {
+            url = "ssh://git@github.com/nixos/nix.git";
+            ref = "master";
+          }
+          ```
+
+          > **Note**
+          > 
+          > Nix will refetch the branch in accordance with
+          > the option `tarball-ttl`.
+
+          > **Note**
+          > 
+          > This behavior is disabled in *Pure evaluation mode*.
+    )",
+    .fun = prim_fetchGit,
+});
 
 }

src/libexpr/primops/fromTOML.cc

@@ -88,6 +88,6 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     }
 }
 
-static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
+static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML);
 
 }

src/libexpr/symbol-table.hh

@@ -28,6 +28,12 @@ public:
         return s == s2.s;
     }
 
+    // FIXME: remove
+    bool operator == (std::string_view s2) const
+    {
+        return s->compare(s2) == 0;
+    }
+
     bool operator != (const Symbol & s2) const
     {
         return s != s2.s;
@@ -68,9 +74,10 @@ private:
     Symbols symbols;
 
 public:
-    Symbol create(const string & s)
+    Symbol create(std::string_view s)
     {
-        std::pair<Symbols::iterator, bool> res = symbols.insert(s);
+        // FIXME: avoid allocation if 's' already exists in the symbol table.
+        std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s));
         return Symbol(&*res.first);
     }
 

src/libexpr/value.hh

@@ -166,6 +166,13 @@ struct Value
     {
         return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
     }
+
+    /* Check whether forcing this value requires a trivial amount of
+       computation. In particular, function applications are
+       non-trivial. */
+    bool isTrivial() const;
+
+    std::vector<std::pair<Path, std::string>> getContext();
 };
 
 

src/libfetchers/attrs.cc

@@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs)
 {
     nlohmann::json json;
     for (auto & attr : attrs) {
-        if (auto v = std::get_if<int64_t>(&attr.second)) {
+        if (auto v = std::get_if<uint64_t>(&attr.second)) {
             json[attr.first] = *v;
         } else if (auto v = std::get_if<std::string>(&attr.second)) {
             json[attr.first] = *v;
@@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name)
     return *s;
 }
 
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
+std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
 {
     auto i = attrs.find(name);
     if (i == attrs.end()) return {};
-    if (auto v = std::get_if<int64_t>(&i->second))
+    if (auto v = std::get_if<uint64_t>(&i->second))
         return *v;
     throw Error("input attribute '%s' is not an integer", name);
 }
 
-int64_t getIntAttr(const Attrs & attrs, const std::string & name)
+uint64_t getIntAttr(const Attrs & attrs, const std::string & name)
 {
     auto s = maybeGetIntAttr(attrs, name);
     if (!s)
@@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na
 {
     auto i = attrs.find(name);
     if (i == attrs.end()) return {};
-    if (auto v = std::get_if<int64_t>(&i->second))
-        return *v;
+    if (auto v = std::get_if<Explicit<bool>>(&i->second))
+        return v->t;
     throw Error("input attribute '%s' is not a Boolean", name);
 }
 
@@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
 {
     std::map<std::string, std::string> query;
     for (auto & attr : attrs) {
-        if (auto v = std::get_if<int64_t>(&attr.second)) {
+        if (auto v = std::get_if<uint64_t>(&attr.second)) {
             query.insert_or_assign(attr.first, fmt("%d", *v));
         } else if (auto v = std::get_if<std::string>(&attr.second)) {
             query.insert_or_assign(attr.first, *v);

src/libfetchers/attrs.hh

@@ -13,9 +13,14 @@ namespace nix::fetchers {
 template<typename T>
 struct Explicit {
     T t;
+
+    bool operator ==(const Explicit<T> & other) const
+    {
+        return t == other.t;
+    }
 };
 
-typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
+typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
 typedef std::map<std::string, Attr> Attrs;
 
 Attrs jsonToAttrs(const nlohmann::json & json);
@@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
 
 std::string getStrAttr(const Attrs & attrs, const std::string & name);
 
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
+std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
 
-int64_t getIntAttr(const Attrs & attrs, const std::string & name);
+uint64_t getIntAttr(const Attrs & attrs, const std::string & name);
 
 std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
 

src/libfetchers/fetchers.cc

@@ -5,71 +5,276 @@
 
 namespace nix::fetchers {
 
-std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
+std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
 
-void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
+void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
 {
-    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
+    if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
     inputSchemes->push_back(std::move(inputScheme));
 }
 
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
+Input Input::fromURL(const std::string & url)
+{
+    return fromURL(parseURL(url));
+}
+
+static void fixupInput(Input & input)
+{
+    // Check common attributes.
+    input.getType();
+    input.getRef();
+    if (input.getRev())
+        input.immutable = true;
+    input.getRevCount();
+    input.getLastModified();
+    if (input.getNarHash())
+        input.immutable = true;
+}
+
+Input Input::fromURL(const ParsedURL & url)
 {
     for (auto & inputScheme : *inputSchemes) {
         auto res = inputScheme->inputFromURL(url);
-        if (res) return res;
+        if (res) {
+            res->scheme = inputScheme;
+            fixupInput(*res);
+            return std::move(*res);
+        }
     }
+
     throw Error("input '%s' is unsupported", url.url);
 }
 
-std::unique_ptr<Input> inputFromURL(const std::string & url)
+Input Input::fromAttrs(Attrs && attrs)
 {
-    return inputFromURL(parseURL(url));
-}
-
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
-{
-    auto attrs2(attrs);
-    attrs2.erase("narHash");
     for (auto & inputScheme : *inputSchemes) {
-        auto res = inputScheme->inputFromAttrs(attrs2);
+        auto res = inputScheme->inputFromAttrs(attrs);
         if (res) {
-            if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
-                // FIXME: require SRI hash.
-                res->narHash = newHashAllowEmpty(*narHash, {});
-            return res;
+            res->scheme = inputScheme;
+            fixupInput(*res);
+            return std::move(*res);
         }
     }
-    throw Error("input '%s' is unsupported", attrsToJson(attrs));
+
+    Input input;
+    input.attrs = attrs;
+    fixupInput(input);
+    return input;
+}
+
+ParsedURL Input::toURL() const
+{
+    if (!scheme)
+        throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
+    return scheme->toURL(*this);
+}
+
+std::string Input::toURLString(const std::map<std::string, std::string> & extraQuery) const
+{
+    auto url = toURL();
+    for (auto & attr : extraQuery)
+        url.query.insert(attr);
+    return url.to_string();
+}
+
+std::string Input::to_string() const
+{
+    return toURL().to_string();
 }
 
 Attrs Input::toAttrs() const
 {
-    auto attrs = toAttrsInternal();
-    if (narHash)
-        attrs.emplace("narHash", narHash->to_string(SRI, true));
-    attrs.emplace("type", type());
     return attrs;
 }
 
-std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
+bool Input::hasAllInfo() const
 {
-    auto [tree, input] = fetchTreeInternal(store);
+    return getNarHash() && scheme && scheme->hasAllInfo(*this);
+}
+
+bool Input::operator ==(const Input & other) const
+{
+    return attrs == other.attrs;
+}
+
+bool Input::contains(const Input & other) const
+{
+    if (*this == other) return true;
+    auto other2(other);
+    other2.attrs.erase("ref");
+    other2.attrs.erase("rev");
+    if (*this == other2) return true;
+    return false;
+}
+
+std::pair<Tree, Input> Input::fetch(ref<Store> store) const
+{
+    if (!scheme)
+        throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
+
+    /* The tree may already be in the Nix store, or it could be
+       substituted (which is often faster than fetching from the
+       original source). So check that. */
+    if (hasAllInfo()) {
+        try {
+            auto storePath = computeStorePath(*store);
+
+            store->ensurePath(storePath);
+
+            debug("using substituted/cached input '%s' in '%s'",
+                to_string(), store->printStorePath(storePath));
+
+            auto actualPath = store->toRealPath(storePath);
+
+            return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this};
+        } catch (Error & e) {
+            debug("substitution of input '%s' failed: %s", to_string(), e.what());
+        }
+    }
+
+    auto [tree, input] = scheme->fetch(store, *this);
 
     if (tree.actualPath == "")
         tree.actualPath = store->toRealPath(tree.storePath);
 
-    if (!tree.info.narHash)
-        tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
+    auto narHash = store->queryPathInfo(tree.storePath)->narHash;
+    input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
 
-    if (input->narHash)
-        assert(input->narHash == tree.info.narHash);
+    if (auto prevNarHash = getNarHash()) {
+        if (narHash != *prevNarHash)
+            throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+                to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
+    }
 
-    if (narHash && narHash != input->narHash)
-        throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
-            to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
+    if (auto prevLastModified = getLastModified()) {
+        if (input.getLastModified() != prevLastModified)
+            throw Error("'lastModified' attribute mismatch in input '%s', expected %d",
+                input.to_string(), *prevLastModified);
+    }
+
+    if (auto prevRevCount = getRevCount()) {
+        if (input.getRevCount() != prevRevCount)
+            throw Error("'revCount' attribute mismatch in input '%s', expected %d",
+                input.to_string(), *prevRevCount);
+    }
+
+    input.immutable = true;
+
+    assert(input.hasAllInfo());
 
     return {std::move(tree), input};
 }
 
+Input Input::applyOverrides(
+    std::optional<std::string> ref,
+    std::optional<Hash> rev) const
+{
+    if (!scheme) return *this;
+    return scheme->applyOverrides(*this, ref, rev);
+}
+
+void Input::clone(const Path & destDir) const
+{
+    assert(scheme);
+    scheme->clone(*this, destDir);
+}
+
+std::optional<Path> Input::getSourcePath() const
+{
+    assert(scheme);
+    return scheme->getSourcePath(*this);
+}
+
+void Input::markChangedFile(
+    std::string_view file,
+    std::optional<std::string> commitMsg) const
+{
+    assert(scheme);
+    return scheme->markChangedFile(*this, file, commitMsg);
+}
+
+StorePath Input::computeStorePath(Store & store) const
+{
+    auto narHash = getNarHash();
+    if (!narHash)
+        throw Error("cannot compute store path for mutable input '%s'", to_string());
+    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+}
+
+std::string Input::getType() const
+{
+    return getStrAttr(attrs, "type");
+}
+
+std::optional<Hash> Input::getNarHash() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "narHash")) {
+        auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
+        if (hash.type != htSHA256)
+            throw UsageError("narHash must use SHA-256");
+        return hash;
+    }
+    return {};
+}
+
+std::optional<std::string> Input::getRef() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "ref"))
+        return *s;
+    return {};
+}
+
+std::optional<Hash> Input::getRev() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "rev"))
+        return Hash::parseAny(*s, htSHA1);
+    return {};
+}
+
+std::optional<uint64_t> Input::getRevCount() const
+{
+    if (auto n = maybeGetIntAttr(attrs, "revCount"))
+        return *n;
+    return {};
+}
+
+std::optional<time_t> Input::getLastModified() const
+{
+    if (auto n = maybeGetIntAttr(attrs, "lastModified"))
+        return *n;
+    return {};
+}
+
+ParsedURL InputScheme::toURL(const Input & input)
+{
+    throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
+}
+
+Input InputScheme::applyOverrides(
+    const Input & input,
+    std::optional<std::string> ref,
+    std::optional<Hash> rev)
+{
+    if (ref)
+        throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref);
+    if (rev)
+        throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev());
+    return input;
+}
+
+std::optional<Path> InputScheme::getSourcePath(const Input & input)
+{
+    return {};
+}
+
+void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
+{
+    assert(false);
+}
+
+void InputScheme::clone(const Input & input, const Path & destDir)
+{
+    throw Error("do not know how to clone input '%s'", input.to_string());
+}
+
 }

src/libfetchers/fetchers.hh

@@ -3,7 +3,6 @@
 #include "types.hh"
 #include "hash.hh"
 #include "path.hh"
-#include "tree-info.hh"
 #include "attrs.hh"
 #include "url.hh"
 
@@ -13,73 +12,106 @@ namespace nix { class Store; }
 
 namespace nix::fetchers {
 
-struct Input;
-
 struct Tree
 {
     Path actualPath;
     StorePath storePath;
-    TreeInfo info;
+    Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {}
 };
 
-struct Input : std::enable_shared_from_this<Input>
+struct InputScheme;
+
+struct Input
 {
-    std::optional<Hash> narHash; // FIXME: implement
+    friend struct InputScheme;
 
-    virtual std::string type() const = 0;
+    std::shared_ptr<InputScheme> scheme; // note: can be null
+    Attrs attrs;
+    bool immutable = false;
+    bool direct = true;
 
-    virtual ~Input() { }
+public:
+    static Input fromURL(const std::string & url);
 
-    virtual bool operator ==(const Input & other) const { return false; }
+    static Input fromURL(const ParsedURL & url);
 
-    /* Check whether this is a "direct" input, that is, not
-       one that goes through a registry. */
-    virtual bool isDirect() const { return true; }
+    static Input fromAttrs(Attrs && attrs);
 
-    /* Check whether this is an "immutable" input, that is,
-       one that contains a commit hash or content hash. */
-    virtual bool isImmutable() const { return (bool) narHash; }
+    ParsedURL toURL() const;
 
-    virtual bool contains(const Input & other) const { return false; }
+    std::string toURLString(const std::map<std::string, std::string> & extraQuery = {}) const;
 
-    virtual std::optional<std::string> getRef() const { return {}; }
-
-    virtual std::optional<Hash> getRev() const { return {}; }
-
-    virtual ParsedURL toURL() const = 0;
-
-    std::string to_string() const
-    {
-        return toURL().to_string();
-    }
+    std::string to_string() const;
 
     Attrs toAttrs() const;
 
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
+    /* Check whether this is a "direct" input, that is, not
+       one that goes through a registry. */
+    bool isDirect() const { return direct; }
 
-private:
+    /* Check whether this is an "immutable" input, that is,
+       one that contains a commit hash or content hash. */
+    bool isImmutable() const { return immutable; }
 
-    virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
+    bool hasAllInfo() const;
 
-    virtual Attrs toAttrsInternal() const = 0;
+    bool operator ==(const Input & other) const;
+
+    bool contains(const Input & other) const;
+
+    std::pair<Tree, Input> fetch(ref<Store> store) const;
+
+    Input applyOverrides(
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) const;
+
+    void clone(const Path & destDir) const;
+
+    std::optional<Path> getSourcePath() const;
+
+    void markChangedFile(
+        std::string_view file,
+        std::optional<std::string> commitMsg) const;
+
+    StorePath computeStorePath(Store & store) const;
+
+    // Convenience functions for common attributes.
+    std::string getType() const;
+    std::optional<Hash> getNarHash() const;
+    std::optional<std::string> getRef() const;
+    std::optional<Hash> getRev() const;
+    std::optional<uint64_t> getRevCount() const;
+    std::optional<time_t> getLastModified() const;
 };
 
 struct InputScheme
 {
-    virtual ~InputScheme() { }
+    virtual ~InputScheme()
+    { }
 
-    virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
+    virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
 
-    virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
+    virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
+
+    virtual ParsedURL toURL(const Input & input);
+
+    virtual bool hasAllInfo(const Input & input) = 0;
+
+    virtual Input applyOverrides(
+        const Input & input,
+        std::optional<std::string> ref,
+        std::optional<Hash> rev);
+
+    virtual void clone(const Input & input, const Path & destDir);
+
+    virtual std::optional<Path> getSourcePath(const Input & input);
+
+    virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
+
+    virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0;
 };
 
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
-
-std::unique_ptr<Input> inputFromURL(const std::string & url);
-
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
-
-void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
+void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
 
 struct DownloadFileResult
 {
@@ -92,12 +124,14 @@ DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool immutable);
+    bool immutable,
+    const Headers & headers = {});
 
-Tree downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool immutable);
+    bool immutable,
+    const Headers & headers = {});
 
 }

src/libfetchers/git.cc

@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <sys/time.h>
 
@@ -22,80 +23,152 @@ static bool isNotDotGitDirectory(const Path & path)
     return not std::regex_match(path, gitDirRegex);
 }
 
-struct GitInput : Input
+struct GitInputScheme : InputScheme
 {
-    ParsedURL url;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
-    bool shallow = false;
-    bool submodules = false;
-
-    GitInput(const ParsedURL & url) : url(url)
-    { }
-
-    std::string type() const override { return "git"; }
-
-    bool operator ==(const Input & other) const override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        auto other2 = dynamic_cast<const GitInput *>(&other);
-        return
-            other2
-            && url == other2->url
-            && rev == other2->rev
-            && ref == other2->ref;
-    }
+        if (url.scheme != "git" &&
+            url.scheme != "git+http" &&
+            url.scheme != "git+https" &&
+            url.scheme != "git+ssh" &&
+            url.scheme != "git+file") return {};
 
-    bool isImmutable() const override
-    {
-        return (bool) rev || narHash;
-    }
+        auto url2(url);
+        if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
+        url2.query.clear();
 
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    ParsedURL toURL() const override
-    {
-        ParsedURL url2(url);
-        if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
-        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
-        if (ref) url2.query.insert_or_assign("ref", *ref);
-        if (shallow) url2.query.insert_or_assign("shallow", "1");
-        return url2;
-    }
-
-    Attrs toAttrsInternal() const override
-    {
         Attrs attrs;
-        attrs.emplace("url", url.to_string());
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        if (shallow)
-            attrs.emplace("shallow", true);
-        if (submodules)
-            attrs.emplace("submodules", true);
-        return attrs;
+        attrs.emplace("type", "git");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else if (name == "shallow")
+                attrs.emplace(name, Explicit<bool> { value == "1" });
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
     }
 
-    std::pair<bool, std::string> getActualUrl() const
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "git") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
+                throw Error("unsupported Git input attribute '%s'", name);
+
+        parseURL(getStrAttr(attrs, "url"));
+        maybeGetBoolAttr(attrs, "shallow");
+        maybeGetBoolAttr(attrs, "submodules");
+
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (std::regex_search(*ref, badGitRefRegex))
+                throw BadURL("invalid Git branch/tag name '%s'", *ref);
+        }
+
+        Input input;
+        input.attrs = attrs;
+        return input;
+    }
+
+    ParsedURL toURL(const Input & input) override
+    {
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        if (url.scheme != "git") url.scheme = "git+" + url.scheme;
+        if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
+        if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
+        if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
+            url.query.insert_or_assign("shallow", "1");
+        return url;
+    }
+
+    bool hasAllInfo(const Input & input) override
+    {
+        bool maybeDirty = !input.getRef();
+        bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
+        return
+            maybeGetIntAttr(input.attrs, "lastModified")
+            && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
+    }
+
+    Input applyOverrides(
+        const Input & input,
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) override
+    {
+        auto res(input);
+        if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) res.attrs.insert_or_assign("ref", *ref);
+        if (!res.getRef() && res.getRev())
+            throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string());
+        return res;
+    }
+
+    void clone(const Input & input, const Path & destDir) override
+    {
+        auto [isLocal, actualUrl] = getActualUrl(input);
+
+        Strings args = {"clone"};
+
+        args.push_back(actualUrl);
+
+        if (auto ref = input.getRef()) {
+            args.push_back("--branch");
+            args.push_back(*ref);
+        }
+
+        if (input.getRev()) throw UnimplementedError("cloning a specific revision is not implemented");
+
+        args.push_back(destDir);
+
+        runProgram("git", true, args);
+    }
+
+    std::optional<Path> getSourcePath(const Input & input) override
+    {
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        if (url.scheme == "file" && !input.getRef() && !input.getRev())
+            return url.path;
+        return {};
+    }
+
+    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
+    {
+        auto sourcePath = getSourcePath(input);
+        assert(sourcePath);
+
+        runProgram("git", true,
+            { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) });
+
+        if (commitMsg)
+            runProgram("git", true,
+                { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
+    }
+
+    std::pair<bool, std::string> getActualUrl(const Input & input) const
     {
         // Don't clone file:// URIs (but otherwise treat them the
         // same as remote URIs, i.e. don't use the working tree or
         // HEAD).
         static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isLocal = url.scheme == "file" && !forceHttp;
         return {isLocal, isLocal ? url.path : url.base};
     }
 
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
         auto name = "source";
 
-        auto input = std::make_shared<GitInput>(*this);
+        Input input(_input);
 
-        assert(!rev || rev->type == htSHA1);
+        bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
+        bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
 
         std::string cacheType = "git";
         if (shallow) cacheType += "-shallow";
@@ -106,39 +179,35 @@ struct GitInput : Input
             return Attrs({
                 {"type", cacheType},
                 {"name", name},
-                {"rev", input->rev->gitRev()},
+                {"rev", input.getRev()->gitRev()},
             });
         };
 
         auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<Tree, std::shared_ptr<const Input>>
+            -> std::pair<Tree, Input>
         {
-            assert(input->rev);
-            assert(!rev || rev == input->rev);
+            assert(input.getRev());
+            assert(!_input.getRev() || _input.getRev() == input.getRev());
+            if (!shallow)
+                input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
+            input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
             return {
-                Tree {
-                    .actualPath = store->toRealPath(storePath),
-                    .storePath = std::move(storePath),
-                    .info = TreeInfo {
-                        .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
-                        .lastModified = getIntAttr(infoAttrs, "lastModified"),
-                    },
-                },
+                Tree(store->toRealPath(storePath), std::move(storePath)),
                 input
             };
         };
 
-        if (rev) {
+        if (input.getRev()) {
             if (auto res = getCache()->lookup(store, getImmutableAttrs()))
                 return makeResult(res->first, std::move(res->second));
         }
 
-        auto [isLocal, actualUrl_] = getActualUrl();
+        auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
         // If this is a local directory and no ref or revision is
         // given, then allow the use of an unclean working tree.
-        if (!input->ref && !input->rev && isLocal) {
+        if (!input.getRef() && !input.getRev() && isLocal) {
             bool clean = false;
 
             /* Check whether this repo has any commits. There are
@@ -197,44 +266,44 @@ struct GitInput : Input
 
                 auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
-                auto tree = Tree {
-                    .actualPath = store->printStorePath(storePath),
-                    .storePath = std::move(storePath),
-                    .info = TreeInfo {
-                        // FIXME: maybe we should use the timestamp of the last
-                        // modified dirty file?
-                        .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
-                    }
-                };
+                // FIXME: maybe we should use the timestamp of the last
+                // modified dirty file?
+                input.attrs.insert_or_assign(
+                    "lastModified",
+                    haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
 
-                return {std::move(tree), input};
+                return {
+                    Tree(store->printStorePath(storePath), std::move(storePath)),
+                    input
+                };
             }
         }
 
-        if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
+        if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
 
         Attrs mutableAttrs({
             {"type", cacheType},
             {"name", name},
             {"url", actualUrl},
-            {"ref", *input->ref},
+            {"ref", *input.getRef()},
         });
 
         Path repoDir;
 
         if (isLocal) {
 
-            if (!input->rev)
-                input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
+            if (!input.getRev())
+                input.attrs.insert_or_assign("rev",
+                    Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
 
             repoDir = actualUrl;
 
         } else {
 
             if (auto res = getCache()->lookup(store, mutableAttrs)) {
-                auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
-                if (!rev || rev == rev2) {
-                    input->rev = rev2;
+                auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
+                if (!input.getRev() || input.getRev() == rev2) {
+                    input.attrs.insert_or_assign("rev", rev2.gitRev());
                     return makeResult(res->first, std::move(res->second));
                 }
             }
@@ -248,18 +317,18 @@ struct GitInput : Input
             }
 
             Path localRefFile =
-                input->ref->compare(0, 5, "refs/") == 0
-                ? cacheDir + "/" + *input->ref
-                : cacheDir + "/refs/heads/" + *input->ref;
+                input.getRef()->compare(0, 5, "refs/") == 0
+                ? cacheDir + "/" + *input.getRef()
+                : cacheDir + "/refs/heads/" + *input.getRef();
 
             bool doFetch;
             time_t now = time(0);
 
             /* If a rev was specified, we need to fetch if it's not in the
                repo. */
-            if (input->rev) {
+            if (input.getRev()) {
                 try {
-                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
+                    runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
                     doFetch = false;
                 } catch (ExecError & e) {
                     if (WIFEXITED(e.status)) {
@@ -282,9 +351,10 @@ struct GitInput : Input
                 // FIXME: git stderr messes up our progress indicator, so
                 // we're using --quiet for now. Should process its stderr.
                 try {
-                    auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
-                        ? *input->ref
-                        : "refs/heads/" + *input->ref;
+                    auto ref = input.getRef();
+                    auto fetchRef = ref->compare(0, 5, "refs/") == 0
+                        ? *ref
+                        : "refs/heads/" + *ref;
                     runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
@@ -300,8 +370,8 @@ struct GitInput : Input
                 utimes(localRefFile.c_str(), times);
             }
 
-            if (!input->rev)
-                input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
+            if (!input.getRev())
+                input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
         }
 
         bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@@ -311,7 +381,7 @@ struct GitInput : Input
 
         // FIXME: check whether rev is an ancestor of ref.
 
-        printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
+        printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
 
         /* Now that we know the ref, check again whether we have it in
            the store. */
@@ -333,7 +403,7 @@ struct GitInput : Input
             runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
                                       "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
 
-            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
+            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
             runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
             runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
 
@@ -342,7 +412,7 @@ struct GitInput : Input
             // FIXME: should pipe this, or find some better way to extract a
             // revision.
             auto source = sinkToSource([&](Sink & sink) {
-                RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
+                RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
                 gitOptions.standardOut = &sink;
                 runProgram2(gitOptions);
             });
@@ -352,18 +422,18 @@ struct GitInput : Input
 
         auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
 
-        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
+        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
 
         Attrs infoAttrs({
-            {"rev", input->rev->gitRev()},
+            {"rev", input.getRev()->gitRev()},
             {"lastModified", lastModified},
         });
 
         if (!shallow)
             infoAttrs.insert_or_assign("revCount",
-                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
+                std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
 
-        if (!this->rev)
+        if (!_input.getRev())
             getCache()->add(
                 store,
                 mutableAttrs,
@@ -382,60 +452,6 @@ struct GitInput : Input
     }
 };
 
-struct GitInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "git" &&
-            url.scheme != "git+http" &&
-            url.scheme != "git+https" &&
-            url.scheme != "git+ssh" &&
-            url.scheme != "git+file") return nullptr;
-
-        auto url2(url);
-        if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
-        url2.query.clear();
-
-        Attrs attrs;
-        attrs.emplace("type", "git");
-
-        for (auto &[name, value] : url.query) {
-            if (name == "rev" || name == "ref")
-                attrs.emplace(name, value);
-            else
-                url2.query.emplace(name, value);
-        }
-
-        attrs.emplace("url", url2.to_string());
-
-        return inputFromAttrs(attrs);
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "git") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
-                throw Error("unsupported Git input attribute '%s'", name);
-
-        auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
-            if (std::regex_search(*ref, badGitRefRegex))
-                throw BadURL("invalid Git branch/tag name '%s'", *ref);
-            input->ref = *ref;
-        }
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
-
-        input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
-
-        input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
-
-        return input;
-    }
-};
-
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
+static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
 
 }

src/libfetchers/github.cc

@@ -3,86 +3,180 @@
 #include "fetchers.hh"
 #include "globals.hh"
 #include "store-api.hh"
+#include "types.hh"
+#include "url-parts.hh"
 
+#include <optional>
 #include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
 
-std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
-std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
-
-struct GitHubInput : Input
+struct DownloadUrl
 {
-    std::string owner;
-    std::string repo;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
+    std::string url;
+    Headers headers;
+};
 
-    std::string type() const override { return "github"; }
+// A github or gitlab host
+const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
+std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
 
-    bool operator ==(const Input & other) const override
+struct GitArchiveInputScheme : InputScheme
+{
+    virtual std::string type() = 0;
+
+    virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
+
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        auto other2 = dynamic_cast<const GitHubInput *>(&other);
-        return
-            other2
-            && owner == other2->owner
-            && repo == other2->repo
-            && rev == other2->rev
-            && ref == other2->ref;
+        if (url.scheme != type()) return {};
+
+        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+
+        std::optional<Hash> rev;
+        std::optional<std::string> ref;
+        std::optional<std::string> host_url;
+
+        if (path.size() == 2) {
+        } else if (path.size() == 3) {
+            if (std::regex_match(path[2], revRegex))
+                rev = Hash::parseAny(path[2], htSHA1);
+            else if (std::regex_match(path[2], refRegex))
+                ref = path[2];
+            else
+                throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
+        } else
+            throw BadURL("URL '%s' is invalid", url.url);
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev") {
+                if (rev)
+                    throw BadURL("URL '%s' contains multiple commit hashes", url.url);
+                rev = Hash::parseAny(value, htSHA1);
+            }
+            else if (name == "ref") {
+                if (!std::regex_match(value, refRegex))
+                    throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
+                if (ref)
+                    throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
+                ref = value;
+            }
+            else if (name == "host") {
+                if (!std::regex_match(value, hostRegex))
+                    throw BadURL("URL '%s' contains an invalid instance host", url.url);
+                host_url = value;
+            }
+            // FIXME: barf on unsupported attributes
+        }
+
+        if (ref && rev)
+            throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
+
+        Input input;
+        input.attrs.insert_or_assign("type", type());
+        input.attrs.insert_or_assign("owner", path[0]);
+        input.attrs.insert_or_assign("repo", path[1]);
+        if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) input.attrs.insert_or_assign("ref", *ref);
+        if (host_url) input.attrs.insert_or_assign("host", *host_url);
+
+        return input;
     }
 
-    bool isImmutable() const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
-        return (bool) rev || narHash;
+        if (maybeGetStrAttr(attrs, "type") != type()) return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
+                throw Error("unsupported input attribute '%s'", name);
+
+        getStrAttr(attrs, "owner");
+        getStrAttr(attrs, "repo");
+
+        Input input;
+        input.attrs = attrs;
+        return input;
     }
 
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    ParsedURL toURL() const override
+    ParsedURL toURL(const Input & input) override
     {
+        auto owner = getStrAttr(input.attrs, "owner");
+        auto repo = getStrAttr(input.attrs, "repo");
+        auto ref = input.getRef();
+        auto rev = input.getRev();
         auto path = owner + "/" + repo;
         assert(!(ref && rev));
         if (ref) path += "/" + *ref;
         if (rev) path += "/" + rev->to_string(Base16, false);
         return ParsedURL {
-            .scheme = "github",
+            .scheme = type(),
             .path = path,
         };
     }
 
-    Attrs toAttrsInternal() const override
+    bool hasAllInfo(const Input & input) override
     {
-        Attrs attrs;
-        attrs.emplace("owner", owner);
-        attrs.emplace("repo", repo);
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        return attrs;
+        return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified");
     }
 
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    Input applyOverrides(
+        const Input & _input,
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) override
     {
-        auto rev = this->rev;
-        auto ref = this->ref.value_or("master");
-
-        if (!rev) {
-            auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
-                owner, repo, ref);
-            auto json = nlohmann::json::parse(
-                readFile(
-                    store->toRealPath(
-                        downloadFile(store, url, "source", false).storePath)));
-            rev = Hash(std::string { json["sha"] }, htSHA1);
-            debug("HEAD revision for '%s' is %s", url, rev->gitRev());
+        auto input(_input);
+        if (rev && ref)
+            throw BadURL("cannot apply both a commit hash (%s) and a branch/tag name ('%s') to input '%s'",
+                rev->gitRev(), *ref, input.to_string());
+        if (rev) {
+            input.attrs.insert_or_assign("rev", rev->gitRev());
+            input.attrs.erase("ref");
         }
+        if (ref) {
+            input.attrs.insert_or_assign("ref", *ref);
+            input.attrs.erase("rev");
+        }
+        return input;
+    }
 
-        auto input = std::make_shared<GitHubInput>(*this);
-        input->ref = {};
-        input->rev = *rev;
+    std::optional<std::string> getAccessToken(const std::string & host) const
+    {
+        auto tokens = settings.accessTokens.get();
+        if (auto token = get(tokens, host))
+            return *token;
+        return {};
+    }
+
+    Headers makeHeadersWithAuthTokens(const std::string & host) const
+    {
+        Headers headers;
+        auto accessToken = getAccessToken(host);
+        if (accessToken) {
+            auto hdr = accessHeaderFromToken(*accessToken);
+            if (hdr)
+                headers.push_back(*hdr);
+            else
+                warn("Unrecognized access token for host '%s'", host);
+        }
+        return headers;
+    }
+
+    virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
+
+    virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
+
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
+    {
+        Input input(_input);
+
+        if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD");
+
+        auto rev = input.getRev();
+        if (!rev) rev = getRevFromRef(store, input);
+
+        input.attrs.erase("ref");
+        input.attrs.insert_or_assign("rev", rev->gitRev());
 
         Attrs immutableAttrs({
             {"type", "git-tarball"},
@@ -90,36 +184,25 @@ struct GitHubInput : Input
         });
 
         if (auto res = getCache()->lookup(store, immutableAttrs)) {
+            input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
             return {
-                Tree{
-                    .actualPath = store->toRealPath(res->second),
-                    .storePath = std::move(res->second),
-                    .info = TreeInfo {
-                        .lastModified = getIntAttr(res->first, "lastModified"),
-                    },
-                },
+                Tree(store->toRealPath(res->second), std::move(res->second)),
                 input
             };
         }
 
-        // FIXME: use regular /archive URLs instead? api.github.com
-        // might have stricter rate limits.
+        auto url = getDownloadUrl(input);
 
-        auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
-            owner, repo, rev->to_string(Base16, false));
+        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
 
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            url += "?access_token=" + accessToken;
-
-        auto tree = downloadTarball(store, url, "source", true);
+        input.attrs.insert_or_assign("lastModified", lastModified);
 
         getCache()->add(
             store,
             immutableAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", *tree.info.lastModified}
+                {"lastModified", lastModified}
             },
             tree.storePath,
             true);
@@ -128,68 +211,130 @@ struct GitHubInput : Input
     }
 };
 
-struct GitHubInputScheme : InputScheme
+struct GitHubInputScheme : GitArchiveInputScheme
 {
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    std::string type() override { return "github"; }
+
+    std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
-        if (url.scheme != "github") return nullptr;
-
-        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
-        auto input = std::make_unique<GitHubInput>();
-
-        if (path.size() == 2) {
-        } else if (path.size() == 3) {
-            if (std::regex_match(path[2], revRegex))
-                input->rev = Hash(path[2], htSHA1);
-            else if (std::regex_match(path[2], refRegex))
-                input->ref = path[2];
-            else
-                throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
-        } else
-            throw BadURL("GitHub URL '%s' is invalid", url.url);
-
-        for (auto &[name, value] : url.query) {
-            if (name == "rev") {
-                if (input->rev)
-                    throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
-                input->rev = Hash(value, htSHA1);
-            }
-            else if (name == "ref") {
-                if (!std::regex_match(value, refRegex))
-                    throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
-                if (input->ref)
-                    throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
-                input->ref = value;
-            }
-        }
-
-        if (input->ref && input->rev)
-            throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
-
-        input->owner = path[0];
-        input->repo = path[1];
-
-        return input;
+        // Github supports PAT/OAuth2 tokens and HTTP Basic
+        // Authentication.  The former simply specifies the token, the
+        // latter can use the token as the password.  Only the first
+        // is used here. See
+        // https://developer.github.com/v3/#authentication and
+        // https://docs.github.com/en/developers/apps/authorizing-oath-apps
+        return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        if (maybeGetStrAttr(attrs, "type") != "github") return {};
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
+        auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
-                throw Error("unsupported GitHub input attribute '%s'", name);
+        Headers headers = makeHeadersWithAuthTokens(host);
 
-        auto input = std::make_unique<GitHubInput>();
-        input->owner = getStrAttr(attrs, "owner");
-        input->repo = getStrAttr(attrs, "repo");
-        input->ref = maybeGetStrAttr(attrs, "ref");
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
-        return input;
+        auto json = nlohmann::json::parse(
+            readFile(
+                store->toRealPath(
+                    downloadFile(store, url, "source", false, headers).storePath)));
+        auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
+        debug("HEAD revision for '%s' is %s", url, rev.gitRev());
+        return rev;
+    }
+
+    DownloadUrl getDownloadUrl(const Input & input) const override
+    {
+        // FIXME: use regular /archive URLs instead? api.github.com
+        // might have stricter rate limits.
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
+        auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            input.getRev()->to_string(Base16, false));
+
+        Headers headers = makeHeadersWithAuthTokens(host);
+        return DownloadUrl { url, headers };
+    }
+
+    void clone(const Input & input, const Path & destDir) override
+    {
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
+        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+                host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .clone(destDir);
     }
 };
 
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+struct GitLabInputScheme : GitArchiveInputScheme
+{
+    std::string type() override { return "gitlab"; }
+
+    std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
+    {
+        // Gitlab supports 4 kinds of authorization, two of which are
+        // relevant here: OAuth2 and PAT (Private Access Token).  The
+        // user can indicate which token is used by specifying the
+        // token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
+        // If the <TYPE> is unrecognized, this will fall back to
+        // treating this simply has <HDRNAME>:<HDRVAL>.  See
+        // https://docs.gitlab.com/12.10/ee/api/README.html#authentication
+        auto fldsplit = token.find_first_of(':');
+        // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
+        if ("OAuth2" == token.substr(0, fldsplit))
+            return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
+        if ("PAT" == token.substr(0, fldsplit))
+            return std::make_pair("Private-token", token.substr(fldsplit+1));
+        warn("Unrecognized GitLab token type %s",  token.substr(0, fldsplit));
+        return std::nullopt;
+    }
+
+    Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
+    {
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+        // See rate limiting note below
+        auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+
+        Headers headers = makeHeadersWithAuthTokens(host);
+
+        auto json = nlohmann::json::parse(
+            readFile(
+                store->toRealPath(
+                    downloadFile(store, url, "source", false, headers).storePath)));
+        auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
+        debug("HEAD revision for '%s' is %s", url, rev.gitRev());
+        return rev;
+    }
+
+    DownloadUrl getDownloadUrl(const Input & input) const override
+    {
+        // This endpoint has a rate limit threshold that may be
+        // server-specific and vary based whether the user is
+        // authenticated via an accessToken or not, but the usual rate
+        // is 10 reqs/sec/ip-addr.  See
+        // https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+        auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            input.getRev()->to_string(Base16, false));
+
+        Headers headers = makeHeadersWithAuthTokens(host);
+        return DownloadUrl { url, headers };
+    }
+
+    void clone(const Input & input, const Path & destDir) override
+    {
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+        // FIXME: get username somewhere
+        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+                host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .clone(destDir);
+    }
+};
+
+static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
 
 }

src/libfetchers/indirect.cc

@@ -0,0 +1,105 @@
+#include "fetchers.hh"
+#include "url-parts.hh"
+
+namespace nix::fetchers {
+
+std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+
+struct IndirectInputScheme : InputScheme
+{
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
+    {
+        if (url.scheme != "flake") return {};
+
+        auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+
+        std::optional<Hash> rev;
+        std::optional<std::string> ref;
+
+        if (path.size() == 1) {
+        } else if (path.size() == 2) {
+            if (std::regex_match(path[1], revRegex))
+                rev = Hash::parseAny(path[1], htSHA1);
+            else if (std::regex_match(path[1], refRegex))
+                ref = path[1];
+            else
+                throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
+        } else if (path.size() == 3) {
+            if (!std::regex_match(path[1], refRegex))
+                throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
+            ref = path[1];
+            if (!std::regex_match(path[2], revRegex))
+                throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
+            rev = Hash::parseAny(path[2], htSHA1);
+        } else
+            throw BadURL("GitHub URL '%s' is invalid", url.url);
+
+        std::string id = path[0];
+        if (!std::regex_match(id, flakeRegex))
+            throw BadURL("'%s' is not a valid flake ID", id);
+
+        // FIXME: forbid query params?
+
+        Input input;
+        input.direct = false;
+        input.attrs.insert_or_assign("type", "indirect");
+        input.attrs.insert_or_assign("id", id);
+        if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) input.attrs.insert_or_assign("ref", *ref);
+
+        return input;
+    }
+
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
+    {
+        if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
+                throw Error("unsupported indirect input attribute '%s'", name);
+
+        auto id = getStrAttr(attrs, "id");
+        if (!std::regex_match(id, flakeRegex))
+            throw BadURL("'%s' is not a valid flake ID", id);
+
+        Input input;
+        input.direct = false;
+        input.attrs = attrs;
+        return input;
+    }
+
+    ParsedURL toURL(const Input & input) override
+    {
+        ParsedURL url;
+        url.scheme = "flake";
+        url.path = getStrAttr(input.attrs, "id");
+        if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; };
+        if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); };
+        return url;
+    }
+
+    bool hasAllInfo(const Input & input) override
+    {
+        return false;
+    }
+
+    Input applyOverrides(
+        const Input & _input,
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) override
+    {
+        auto input(_input);
+        if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) input.attrs.insert_or_assign("ref", *ref);
+        return input;
+    }
+
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    {
+        throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
+    }
+};
+
+static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
+
+}

src/libfetchers/mercurial.cc

@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "tarfile.hh"
 #include "store-api.hh"
+#include "url-parts.hh"
 
 #include <sys/time.h>
 
@@ -10,76 +11,124 @@ using namespace std::string_literals;
 
 namespace nix::fetchers {
 
-struct MercurialInput : Input
+struct MercurialInputScheme : InputScheme
 {
-    ParsedURL url;
-    std::optional<std::string> ref;
-    std::optional<Hash> rev;
-
-    MercurialInput(const ParsedURL & url) : url(url)
-    { }
-
-    std::string type() const override { return "hg"; }
-
-    bool operator ==(const Input & other) const override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        auto other2 = dynamic_cast<const MercurialInput *>(&other);
-        return
-            other2
-            && url == other2->url
-            && rev == other2->rev
-            && ref == other2->ref;
+        if (url.scheme != "hg+http" &&
+            url.scheme != "hg+https" &&
+            url.scheme != "hg+ssh" &&
+            url.scheme != "hg+file") return {};
+
+        auto url2(url);
+        url2.scheme = std::string(url2.scheme, 3);
+        url2.query.clear();
+
+        Attrs attrs;
+        attrs.emplace("type", "hg");
+
+        for (auto &[name, value] : url.query) {
+            if (name == "rev" || name == "ref")
+                attrs.emplace(name, value);
+            else
+                url2.query.emplace(name, value);
+        }
+
+        attrs.emplace("url", url2.to_string());
+
+        return inputFromAttrs(attrs);
     }
 
-    bool isImmutable() const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
-        return (bool) rev || narHash;
+        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
+
+        for (auto & [name, value] : attrs)
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+                throw Error("unsupported Mercurial input attribute '%s'", name);
+
+        parseURL(getStrAttr(attrs, "url"));
+
+        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+            if (!std::regex_match(*ref, refRegex))
+                throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
+        }
+
+        Input input;
+        input.attrs = attrs;
+        return input;
     }
 
-    std::optional<std::string> getRef() const override { return ref; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    ParsedURL toURL() const override
+    ParsedURL toURL(const Input & input) override
     {
-        ParsedURL url2(url);
-        url2.scheme = "hg+" + url2.scheme;
-        if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
-        if (ref) url2.query.insert_or_assign("ref", *ref);
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        url.scheme = "hg+" + url.scheme;
+        if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
+        if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
         return url;
     }
 
-    Attrs toAttrsInternal() const override
+    bool hasAllInfo(const Input & input) override
     {
-        Attrs attrs;
-        attrs.emplace("url", url.to_string());
-        if (ref)
-            attrs.emplace("ref", *ref);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        return attrs;
+        // FIXME: ugly, need to distinguish between dirty and clean
+        // default trees.
+        return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount");
     }
 
-    std::pair<bool, std::string> getActualUrl() const
+    Input applyOverrides(
+        const Input & input,
+        std::optional<std::string> ref,
+        std::optional<Hash> rev) override
     {
+        auto res(input);
+        if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
+        if (ref) res.attrs.insert_or_assign("ref", *ref);
+        return res;
+    }
+
+    std::optional<Path> getSourcePath(const Input & input) override
+    {
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        if (url.scheme == "file" && !input.getRef() && !input.getRev())
+            return url.path;
+        return {};
+    }
+
+    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
+    {
+        auto sourcePath = getSourcePath(input);
+        assert(sourcePath);
+
+        // FIXME: shut up if file is already tracked.
+        runProgram("hg", true,
+            { "add", *sourcePath + "/" + std::string(file) });
+
+        if (commitMsg)
+            runProgram("hg", true,
+                { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
+    }
+
+    std::pair<bool, std::string> getActualUrl(const Input & input) const
+    {
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isLocal = url.scheme == "file";
         return {isLocal, isLocal ? url.path : url.base};
     }
 
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
         auto name = "source";
 
-        auto input = std::make_shared<MercurialInput>(*this);
+        Input input(_input);
 
-        auto [isLocal, actualUrl_] = getActualUrl();
+        auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
         // FIXME: return lastModified.
 
         // FIXME: don't clone local repositories.
 
-        if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
+        if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
 
             bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
 
@@ -94,7 +143,7 @@ struct MercurialInput : Input
                 if (settings.warnDirty)
                     warn("Mercurial tree '%s' is unclean", actualUrl);
 
-                input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
+                input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
 
                 auto files = tokenizeString<std::set<std::string>>(
                     runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
@@ -116,60 +165,54 @@ struct MercurialInput : Input
 
                 auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
-                return {Tree {
-                    .actualPath = store->printStorePath(storePath),
-                    .storePath = std::move(storePath),
-                }, input};
+                return {
+                    Tree(store->printStorePath(storePath), std::move(storePath)),
+                    input
+                };
             }
         }
 
-        if (!input->ref) input->ref = "default";
+        if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
 
         auto getImmutableAttrs = [&]()
         {
             return Attrs({
                 {"type", "hg"},
                 {"name", name},
-                {"rev", input->rev->gitRev()},
+                {"rev", input.getRev()->gitRev()},
             });
         };
 
         auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<Tree, std::shared_ptr<const Input>>
+            -> std::pair<Tree, Input>
         {
-            assert(input->rev);
-            assert(!rev || rev == input->rev);
+            assert(input.getRev());
+            assert(!_input.getRev() || _input.getRev() == input.getRev());
+            input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
             return {
-                Tree{
-                    .actualPath = store->toRealPath(storePath),
-                    .storePath = std::move(storePath),
-                    .info = TreeInfo {
-                        .revCount = getIntAttr(infoAttrs, "revCount"),
-                    },
-                },
+                Tree(store->toRealPath(storePath), std::move(storePath)),
                 input
             };
         };
 
-        if (input->rev) {
+        if (input.getRev()) {
             if (auto res = getCache()->lookup(store, getImmutableAttrs()))
                 return makeResult(res->first, std::move(res->second));
         }
 
-        assert(input->rev || input->ref);
-        auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
+        auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef();
 
         Attrs mutableAttrs({
             {"type", "hg"},
             {"name", name},
             {"url", actualUrl},
-            {"ref", *input->ref},
+            {"ref", *input.getRef()},
         });
 
         if (auto res = getCache()->lookup(store, mutableAttrs)) {
-            auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
-            if (!rev || rev == rev2) {
-                input->rev = rev2;
+            auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
+            if (!input.getRev() || input.getRev() == rev2) {
+                input.attrs.insert_or_assign("rev", rev2.gitRev());
                 return makeResult(res->first, std::move(res->second));
             }
         }
@@ -178,10 +221,10 @@ struct MercurialInput : Input
 
         /* If this is a commit hash that we already have, we don't
            have to pull again. */
-        if (!(input->rev
+        if (!(input.getRev()
                 && pathExists(cacheDir)
                 && runProgram(
-                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
+                    RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
                     .killStderr(true)).second == "1"))
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
@@ -210,9 +253,9 @@ struct MercurialInput : Input
             runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
         assert(tokens.size() == 3);
 
-        input->rev = Hash(tokens[0], htSHA1);
+        input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
         auto revCount = std::stoull(tokens[1]);
-        input->ref = tokens[2];
+        input.attrs.insert_or_assign("ref", tokens[2]);
 
         if (auto res = getCache()->lookup(store, getImmutableAttrs()))
             return makeResult(res->first, std::move(res->second));
@@ -220,18 +263,18 @@ struct MercurialInput : Input
         Path tmpDir = createTempDir();
         AutoDelete delTmpDir(tmpDir, true);
 
-        runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
+        runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
         auto storePath = store->addToStore(name, tmpDir);
 
         Attrs infoAttrs({
-            {"rev", input->rev->gitRev()},
+            {"rev", input.getRev()->gitRev()},
             {"revCount", (int64_t) revCount},
         });
 
-        if (!this->rev)
+        if (!_input.getRev())
             getCache()->add(
                 store,
                 mutableAttrs,
@@ -250,54 +293,6 @@ struct MercurialInput : Input
     }
 };
 
-struct MercurialInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "hg+http" &&
-            url.scheme != "hg+https" &&
-            url.scheme != "hg+ssh" &&
-            url.scheme != "hg+file") return nullptr;
-
-        auto url2(url);
-        url2.scheme = std::string(url2.scheme, 3);
-        url2.query.clear();
-
-        Attrs attrs;
-        attrs.emplace("type", "hg");
-
-        for (auto &[name, value] : url.query) {
-            if (name == "rev" || name == "ref")
-                attrs.emplace(name, value);
-            else
-                url2.query.emplace(name, value);
-        }
-
-        attrs.emplace("url", url2.to_string());
-
-        return inputFromAttrs(attrs);
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "hg") return {};
-
-        for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev")
-                throw Error("unsupported Mercurial input attribute '%s'", name);
-
-        auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto ref = maybeGetStrAttr(attrs, "ref")) {
-            if (!std::regex_match(*ref, refRegex))
-                throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
-            input->ref = *ref;
-        }
-        if (auto rev = maybeGetStrAttr(attrs, "rev"))
-            input->rev = Hash(*rev, htSHA1);
-        return input;
-    }
-};
-
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
+static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
 
 }

src/libfetchers/path.cc

@@ -3,65 +3,86 @@
 
 namespace nix::fetchers {
 
-struct PathInput : Input
+struct PathInputScheme : InputScheme
 {
-    Path path;
-
-    /* Allow the user to pass in "fake" tree info attributes. This is
-       useful for making a pinned tree work the same as the repository
-       from which is exported
-       (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
-    std::optional<Hash> rev;
-    std::optional<uint64_t> revCount;
-    std::optional<time_t> lastModified;
-
-    std::string type() const override { return "path"; }
-
-    std::optional<Hash> getRev() const override { return rev; }
-
-    bool operator ==(const Input & other) const override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        auto other2 = dynamic_cast<const PathInput *>(&other);
-        return
-            other2
-            && path == other2->path
-            && rev == other2->rev
-            && revCount == other2->revCount
-            && lastModified == other2->lastModified;
+        if (url.scheme != "path") return {};
+
+        if (url.authority && *url.authority != "")
+            throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
+
+        Input input;
+        input.attrs.insert_or_assign("type", "path");
+        input.attrs.insert_or_assign("path", url.path);
+
+        for (auto & [name, value] : url.query)
+            if (name == "rev" || name == "narHash")
+                input.attrs.insert_or_assign(name, value);
+            else if (name == "revCount" || name == "lastModified") {
+                uint64_t n;
+                if (!string2Int(value, n))
+                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
+                input.attrs.insert_or_assign(name, n);
+            }
+            else
+                throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
+
+        return input;
     }
 
-    bool isImmutable() const override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
-        return (bool) narHash;
+        if (maybeGetStrAttr(attrs, "type") != "path") return {};
+
+        getStrAttr(attrs, "path");
+
+        for (auto & [name, value] : attrs)
+            /* Allow the user to pass in "fake" tree info
+               attributes. This is useful for making a pinned tree
+               work the same as the repository from which is exported
+               (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
+            if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
+                // checked in Input::fromAttrs
+                ;
+            else
+                throw Error("unsupported path input attribute '%s'", name);
+
+        Input input;
+        input.attrs = attrs;
+        return input;
     }
 
-    ParsedURL toURL() const override
+    ParsedURL toURL(const Input & input) override
     {
-        auto query = attrsToQuery(toAttrsInternal());
+        auto query = attrsToQuery(input.attrs);
         query.erase("path");
+        query.erase("type");
         return ParsedURL {
             .scheme = "path",
-            .path = path,
+            .path = getStrAttr(input.attrs, "path"),
             .query = query,
         };
     }
 
-    Attrs toAttrsInternal() const override
+    bool hasAllInfo(const Input & input) override
     {
-        Attrs attrs;
-        attrs.emplace("path", path);
-        if (rev)
-            attrs.emplace("rev", rev->gitRev());
-        if (revCount)
-            attrs.emplace("revCount", *revCount);
-        if (lastModified)
-            attrs.emplace("lastModified", *lastModified);
-        return attrs;
+        return true;
     }
 
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+    std::optional<Path> getSourcePath(const Input & input) override
     {
-        auto input = std::make_shared<PathInput>(*this);
+        return getStrAttr(input.attrs, "path");
+    }
+
+    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
+    {
+        // nothing to do
+    }
+
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    {
+        auto path = getStrAttr(input.attrs, "path");
 
         // FIXME: check whether access to 'path' is allowed.
 
@@ -74,75 +95,13 @@ struct PathInput : Input
             // FIXME: try to substitute storePath.
             storePath = store->addToStore("source", path);
 
-        return
-            {
-                Tree {
-                    .actualPath = store->toRealPath(*storePath),
-                    .storePath = std::move(*storePath),
-                    .info = TreeInfo {
-                        .revCount = revCount,
-                        .lastModified = lastModified
-                    }
-                },
-                input
-            };
-    }
-
-};
-
-struct PathInputScheme : InputScheme
-{
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
-    {
-        if (url.scheme != "path") return nullptr;
-
-        auto input = std::make_unique<PathInput>();
-        input->path = url.path;
-
-        for (auto & [name, value] : url.query)
-            if (name == "rev")
-                input->rev = Hash(value, htSHA1);
-            else if (name == "revCount") {
-                uint64_t revCount;
-                if (!string2Int(value, revCount))
-                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
-                input->revCount = revCount;
-            }
-            else if (name == "lastModified") {
-                time_t lastModified;
-                if (!string2Int(value, lastModified))
-                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
-                input->lastModified = lastModified;
-            }
-            else
-                throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
-
-        return input;
-    }
-
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
-    {
-        if (maybeGetStrAttr(attrs, "type") != "path") return {};
-
-        auto input = std::make_unique<PathInput>();
-        input->path = getStrAttr(attrs, "path");
-
-        for (auto & [name, value] : attrs)
-            if (name == "rev")
-                input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
-            else if (name == "revCount")
-                input->revCount = getIntAttr(attrs, "revCount");
-            else if (name == "lastModified")
-                input->lastModified = getIntAttr(attrs, "lastModified");
-            else if (name == "type" || name == "path")
-                ;
-            else
-                throw Error("unsupported path input attribute '%s'", name);
-
-        return input;
+        return {
+            Tree(store->toRealPath(*storePath), std::move(*storePath)),
+            input
+        };
     }
 };
 
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
+static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
 
 }

src/libfetchers/registry.cc

@@ -0,0 +1,212 @@
+#include "registry.hh"
+#include "fetchers.hh"
+#include "util.hh"
+#include "globals.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::shared_ptr<Registry> Registry::read(
+    const Path & path, RegistryType type)
+{
+    auto registry = std::make_shared<Registry>(type);
+
+    if (!pathExists(path))
+        return std::make_shared<Registry>(type);
+
+    try {
+
+        auto json = nlohmann::json::parse(readFile(path));
+
+        auto version = json.value("version", 0);
+
+        if (version == 2) {
+            for (auto & i : json["flakes"]) {
+                auto toAttrs = jsonToAttrs(i["to"]);
+                Attrs extraAttrs;
+                auto j = toAttrs.find("dir");
+                if (j != toAttrs.end()) {
+                    extraAttrs.insert(*j);
+                    toAttrs.erase(j);
+                }
+                auto exact = i.find("exact");
+                registry->entries.push_back(
+                    Entry {
+                        .from = Input::fromAttrs(jsonToAttrs(i["from"])),
+                        .to = Input::fromAttrs(std::move(toAttrs)),
+                        .extraAttrs = extraAttrs,
+                        .exact = exact != i.end() && exact.value()
+                    });
+            }
+        }
+
+        else
+            throw Error("flake registry '%s' has unsupported version %d", path, version);
+
+    } catch (nlohmann::json::exception & e) {
+        warn("cannot parse flake registry '%s': %s", path, e.what());
+    } catch (Error & e) {
+        warn("cannot read flake registry '%s': %s", path, e.what());
+    }
+
+    return registry;
+}
+
+void Registry::write(const Path & path)
+{
+    nlohmann::json arr;
+    for (auto & entry : entries) {
+        nlohmann::json obj;
+        obj["from"] = attrsToJson(entry.from.toAttrs());
+        obj["to"] = attrsToJson(entry.to.toAttrs());
+        if (!entry.extraAttrs.empty())
+            obj["to"].update(attrsToJson(entry.extraAttrs));
+        if (entry.exact)
+            obj["exact"] = true;
+        arr.emplace_back(std::move(obj));
+    }
+
+    nlohmann::json json;
+    json["version"] = 2;
+    json["flakes"] = std::move(arr);
+
+    createDirs(dirOf(path));
+    writeFile(path, json.dump(2));
+}
+
+void Registry::add(
+    const Input & from,
+    const Input & to,
+    const Attrs & extraAttrs)
+{
+    entries.emplace_back(
+        Entry {
+            .from = from,
+            .to = to,
+            .extraAttrs = extraAttrs
+        });
+}
+
+void Registry::remove(const Input & input)
+{
+    // FIXME: use C++20 std::erase.
+    for (auto i = entries.begin(); i != entries.end(); )
+        if (i->from == input)
+            i = entries.erase(i);
+        else
+            ++i;
+}
+
+static Path getSystemRegistryPath()
+{
+    return settings.nixConfDir + "/registry.json";
+}
+
+static std::shared_ptr<Registry> getSystemRegistry()
+{
+    static auto systemRegistry =
+        Registry::read(getSystemRegistryPath(), Registry::System);
+    return systemRegistry;
+}
+
+Path getUserRegistryPath()
+{
+    return getHome() + "/.config/nix/registry.json";
+}
+
+std::shared_ptr<Registry> getUserRegistry()
+{
+    static auto userRegistry =
+        Registry::read(getUserRegistryPath(), Registry::User);
+    return userRegistry;
+}
+
+static std::shared_ptr<Registry> flagRegistry =
+    std::make_shared<Registry>(Registry::Flag);
+
+std::shared_ptr<Registry> getFlagRegistry()
+{
+    return flagRegistry;
+}
+
+void overrideRegistry(
+    const Input & from,
+    const Input & to,
+    const Attrs & extraAttrs)
+{
+    flagRegistry->add(from, to, extraAttrs);
+}
+
+static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
+{
+    static auto reg = [&]() {
+        auto path = settings.flakeRegistry.get();
+
+        if (!hasPrefix(path, "/")) {
+            auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
+            if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
+                store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");
+            path = store->toRealPath(storePath);
+        }
+
+        return Registry::read(path, Registry::Global);
+    }();
+
+    return reg;
+}
+
+Registries getRegistries(ref<Store> store)
+{
+    Registries registries;
+    registries.push_back(getFlagRegistry());
+    registries.push_back(getUserRegistry());
+    registries.push_back(getSystemRegistry());
+    registries.push_back(getGlobalRegistry(store));
+    return registries;
+}
+
+std::pair<Input, Attrs> lookupInRegistries(
+    ref<Store> store,
+    const Input & _input)
+{
+    Attrs extraAttrs;
+    int n = 0;
+    Input input(_input);
+
+ restart:
+
+    n++;
+    if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
+
+    for (auto & registry : getRegistries(store)) {
+        // FIXME: O(n)
+        for (auto & entry : registry->entries) {
+            if (entry.exact) {
+                if (entry.from == input) {
+                    input = entry.to;
+                    extraAttrs = entry.extraAttrs;
+                    goto restart;
+                }
+            } else {
+                if (entry.from.contains(input)) {
+                    input = entry.to.applyOverrides(
+                        !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(),
+                        !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>());
+                    extraAttrs = entry.extraAttrs;
+                    goto restart;
+                }
+            }
+        }
+    }
+
+    if (!input.isDirect())
+        throw Error("cannot find flake '%s' in the flake registries", input.to_string());
+
+    debug("looked up '%s' -> '%s'", _input.to_string(), input.to_string());
+
+    return {input, extraAttrs};
+}
+
+}

src/libfetchers/registry.hh

@@ -0,0 +1,64 @@
+#pragma once
+
+#include "types.hh"
+#include "fetchers.hh"
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct Registry
+{
+    enum RegistryType {
+        Flag = 0,
+        User = 1,
+        System = 2,
+        Global = 3,
+    };
+
+    RegistryType type;
+
+    struct Entry
+    {
+        Input from, to;
+        Attrs extraAttrs;
+        bool exact = false;
+    };
+
+    std::vector<Entry> entries;
+
+    Registry(RegistryType type)
+        : type(type)
+    { }
+
+    static std::shared_ptr<Registry> read(
+        const Path & path, RegistryType type);
+
+    void write(const Path & path);
+
+    void add(
+        const Input & from,
+        const Input & to,
+        const Attrs & extraAttrs);
+
+    void remove(const Input & input);
+};
+
+typedef std::vector<std::shared_ptr<Registry>> Registries;
+
+std::shared_ptr<Registry> getUserRegistry();
+
+Path getUserRegistryPath();
+
+Registries getRegistries(ref<Store> store);
+
+void overrideRegistry(
+    const Input & from,
+    const Input & to,
+    const Attrs & extraAttrs);
+
+std::pair<Input, Attrs> lookupInRegistries(
+    ref<Store> store,
+    const Input & input);
+
+}

src/libfetchers/tarball.cc

@@ -5,6 +5,7 @@
 #include "store-api.hh"
 #include "archive.hh"
 #include "tarfile.hh"
+#include "types.hh"
 
 namespace nix::fetchers {
 
@@ -12,7 +13,8 @@ DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool immutable)
+    bool immutable,
+    const Headers & headers)
 {
     // FIXME: check store
 
@@ -37,6 +39,7 @@ DownloadFileResult downloadFile(
         return useCached();
 
     FileTransferRequest request(url);
+    request.headers = headers;
     if (cached)
         request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
     FileTransferResult res;
@@ -67,8 +70,10 @@ DownloadFileResult downloadFile(
         StringSink sink;
         dumpString(*res.data, sink);
         auto hash = hashString(htSHA256, *res.data);
-        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
-        info.narHash = hashString(htSHA256, *sink.s);
+        ValidPathInfo info {
+            store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
+            hashString(htSHA256, *sink.s),
+        };
         info.narSize = sink.s->size();
         info.ca = FixedOutputHash {
             .method = FileIngestionMethod::Flat,
@@ -105,11 +110,12 @@ DownloadFileResult downloadFile(
     };
 }
 
-Tree downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool immutable)
+    bool immutable,
+    const Headers & headers)
 {
     Attrs inAttrs({
         {"type", "tarball"},
@@ -120,15 +126,12 @@ Tree downloadTarball(
     auto cached = getCache()->lookupExpired(store, inAttrs);
 
     if (cached && !cached->expired)
-        return Tree {
-            .actualPath = store->toRealPath(cached->storePath),
-            .storePath = std::move(cached->storePath),
-            .info = TreeInfo {
-                .lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
-            },
+        return {
+            Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
+            getIntAttr(cached->infoAttrs, "lastModified")
         };
 
-    auto res = downloadFile(store, url, name, immutable);
+    auto res = downloadFile(store, url, name, immutable, headers);
 
     std::optional<StorePath> unpackedStorePath;
     time_t lastModified;
@@ -160,119 +163,74 @@ Tree downloadTarball(
         *unpackedStorePath,
         immutable);
 
-    return Tree {
-        .actualPath = store->toRealPath(*unpackedStorePath),
-        .storePath = std::move(*unpackedStorePath),
-        .info = TreeInfo {
-            .lastModified = lastModified,
-        },
+    return {
+        Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
+        lastModified,
     };
 }
 
-struct TarballInput : Input
-{
-    ParsedURL url;
-    std::optional<Hash> hash;
-
-    TarballInput(const ParsedURL & url) : url(url)
-    { }
-
-    std::string type() const override { return "tarball"; }
-
-    bool operator ==(const Input & other) const override
-    {
-        auto other2 = dynamic_cast<const TarballInput *>(&other);
-        return
-            other2
-            && to_string() == other2->to_string()
-            && hash == other2->hash;
-    }
-
-    bool isImmutable() const override
-    {
-        return hash || narHash;
-    }
-
-    ParsedURL toURL() const override
-    {
-        auto url2(url);
-        // NAR hashes are preferred over file hashes since tar/zip files
-        // don't have a canonical representation.
-        if (narHash)
-            url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
-        else if (hash)
-            url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
-        return url2;
-    }
-
-    Attrs toAttrsInternal() const override
-    {
-        Attrs attrs;
-        attrs.emplace("url", url.to_string());
-        if (hash)
-            attrs.emplace("hash", hash->to_string(SRI, true));
-        return attrs;
-    }
-
-    std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
-    {
-        auto tree = downloadTarball(store, url.to_string(), "source", false);
-
-        auto input = std::make_shared<TarballInput>(*this);
-        input->narHash = store->queryPathInfo(tree.storePath)->narHash;
-
-        return {std::move(tree), input};
-    }
-};
-
 struct TarballInputScheme : InputScheme
 {
-    std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+    std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
+        if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {};
 
         if (!hasSuffix(url.path, ".zip")
             && !hasSuffix(url.path, ".tar")
             && !hasSuffix(url.path, ".tar.gz")
             && !hasSuffix(url.path, ".tar.xz")
             && !hasSuffix(url.path, ".tar.bz2"))
-            return nullptr;
-
-        auto input = std::make_unique<TarballInput>(url);
-
-        auto hash = input->url.query.find("hash");
-        if (hash != input->url.query.end()) {
-            // FIXME: require SRI hash.
-            input->hash = Hash(hash->second);
-            input->url.query.erase(hash);
-        }
-
-        auto narHash = input->url.query.find("narHash");
-        if (narHash != input->url.query.end()) {
-            // FIXME: require SRI hash.
-            input->narHash = Hash(narHash->second);
-            input->url.query.erase(narHash);
-        }
+            return {};
 
+        Input input;
+        input.attrs.insert_or_assign("type", "tarball");
+        input.attrs.insert_or_assign("url", url.to_string());
+        auto narHash = url.query.find("narHash");
+        if (narHash != url.query.end())
+            input.attrs.insert_or_assign("narHash", narHash->second);
         return input;
     }
 
-    std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+    std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
         if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "hash")
+            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
                 throw Error("unsupported tarball input attribute '%s'", name);
 
-        auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
-        if (auto hash = maybeGetStrAttr(attrs, "hash"))
-            input->hash = newHashAllowEmpty(*hash, {});
-
+        Input input;
+        input.attrs = attrs;
+        //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash");
         return input;
     }
+
+    ParsedURL toURL(const Input & input) override
+    {
+        auto url = parseURL(getStrAttr(input.attrs, "url"));
+        // NAR hashes are preferred over file hashes since tar/zip files
+        // don't have a canonical representation.
+        if (auto narHash = input.getNarHash())
+            url.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
+        /*
+        else if (auto hash = maybeGetStrAttr(input.attrs, "hash"))
+            url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true));
+        */
+        return url;
+    }
+
+    bool hasAllInfo(const Input & input) override
+    {
+        return true;
+    }
+
+    std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+    {
+        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+        return {std::move(tree), input};
+    }
 };
 
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
+static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
 
 }

src/libfetchers/tree-info.cc

@@ -1,14 +0,0 @@
-#include "tree-info.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-StorePath TreeInfo::computeStorePath(Store & store) const
-{
-    assert(narHash);
-    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
-}
-
-}

src/libfetchers/tree-info.hh

@@ -1,29 +0,0 @@
-#pragma once
-
-#include "path.hh"
-#include "hash.hh"
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix { class Store; }
-
-namespace nix::fetchers {
-
-struct TreeInfo
-{
-    Hash narHash;
-    std::optional<uint64_t> revCount;
-    std::optional<time_t> lastModified;
-
-    bool operator ==(const TreeInfo & other) const
-    {
-        return
-            narHash == other.narHash
-            && revCount == other.revCount
-            && lastModified == other.lastModified;
-    }
-
-    StorePath computeStorePath(Store & store) const;
-};
-
-}

src/libmain/common-args.cc

@@ -28,21 +28,31 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     addFlag({
         .longName = "option",
-        .description = "set a Nix configuration option (overriding nix.conf)",
+        .description = "set a Nix configuration option (overriding `nix.conf`)",
         .labels = {"name", "value"},
         .handler = {[](std::string name, std::string value) {
             try {
                 globalConfig.set(name, value);
             } catch (UsageError & e) {
-                warn(e.what());
+                if (!completions)
+                    warn(e.what());
             }
         }},
+        .completer = [](size_t index, std::string_view prefix) {
+            if (index == 0) {
+                std::map<std::string, Config::SettingInfo> settings;
+                globalConfig.getSettings(settings);
+                for (auto & s : settings)
+                    if (hasPrefix(s.first, prefix))
+                        completions->insert(s.first);
+            }
+        }
     });
 
     addFlag({
         .longName = "log-format",
-        .description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
-                        "or \"bar-with-logs\"",
+        .description = "format of log output; `raw`, `internal-json`, `bar` "
+                        "or `bar-with-logs`",
         .labels = {"format"},
         .handler = {[](std::string format) { setLogFormat(format); }},
     });

src/libmain/loggers.cc

@@ -1,12 +1,13 @@
 #include "loggers.hh"
 #include "progress-bar.hh"
+#include "util.hh"
 
 namespace nix {
 
 LogFormat defaultLogFormat = LogFormat::raw;
 
 LogFormat parseLogFormat(const std::string & logFormatStr) {
-    if (logFormatStr == "raw")
+    if (logFormatStr == "raw" || getEnv("NIX_GET_COMPLETIONS"))
         return LogFormat::raw;
     else if (logFormatStr == "raw-with-logs")
         return LogFormat::rawWithLogs;

src/libmain/progress-bar.cc

@@ -256,7 +256,7 @@ public:
         }
 
         else if (type == resBuildLogLine || type == resPostBuildLogLine) {
-            auto lastLine = trim(getS(fields, 0));
+            auto lastLine = chomp(getS(fields, 0));
             if (!lastLine.empty()) {
                 auto i = state->its.find(act);
                 assert(i != state->its.end());
@@ -362,7 +362,7 @@ public:
         auto width = getWindowSize().second;
         if (width <= 0) width = std::numeric_limits<decltype(width)>::max();
 
-        writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K");
+        writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
     }
 
     std::string getStatus(State & state)

src/libmain/shared.cc

@@ -36,7 +36,7 @@ void printGCWarning()
 
 void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
 {
-    unsigned long long downloadSize, narSize;
+    uint64_t downloadSize, narSize;
     StorePathSet willBuild, willSubstitute, unknown;
     store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
     printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
@@ -45,7 +45,7 @@ void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & pa
 
 void printMissing(ref<Store> store, const StorePathSet & willBuild,
     const StorePathSet & willSubstitute, const StorePathSet & unknown,
-    unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
+    uint64_t downloadSize, uint64_t narSize, Verbosity lvl)
 {
     if (!willBuild.empty()) {
         if (willBuild.size() == 1)
@@ -277,6 +277,8 @@ void printVersion(const string & programName)
 #if HAVE_SODIUM
         cfg.push_back("signed-caches");
 #endif
+        std::cout << "System type: " << settings.thisSystem << "\n";
+        std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
         std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
         std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
         std::cout << "User configuration files: " <<
@@ -384,18 +386,12 @@ RunPager::~RunPager()
 }
 
 
-string showBytes(unsigned long long bytes)
-{
-    return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
-}
-
-
 PrintFreed::~PrintFreed()
 {
     if (show)
-        std::cout << format("%1% store paths deleted, %2% freed\n")
-            % results.paths.size()
-            % showBytes(results.bytesFreed);
+        std::cout << fmt("%d store paths deleted, %s freed\n",
+            results.paths.size(),
+            showBytes(results.bytesFreed));
 }
 
 Exit::~Exit() { }

src/libmain/shared.hh

@@ -47,7 +47,7 @@ void printMissing(
 
 void printMissing(ref<Store> store, const StorePathSet & willBuild,
     const StorePathSet & willSubstitute, const StorePathSet & unknown,
-    unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
+    uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo);
 
 string getArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end);
@@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt;
 
 /* GC helpers. */
 
-string showBytes(unsigned long long bytes);
+string showBytes(uint64_t bytes);
 
 struct GCResults;
 

src/libstore/binary-cache-store.cc

@@ -11,17 +11,20 @@
 #include "nar-accessor.hh"
 #include "json.hh"
 #include "thread-pool.hh"
+#include "callback.hh"
 
 #include <chrono>
 #include <future>
 #include <regex>
+#include <fstream>
 
 #include <nlohmann/json.hpp>
 
 namespace nix {
 
 BinaryCacheStore::BinaryCacheStore(const Params & params)
-    : Store(params)
+    : BinaryCacheStoreConfig(params)
+    , Store(params)
 {
     if (secretKeyFile != "")
         secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
@@ -57,6 +60,13 @@ void BinaryCacheStore::init()
     }
 }
 
+void BinaryCacheStore::upsertFile(const std::string & path,
+    std::string && data,
+    const std::string & mimeType)
+{
+    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
+}
+
 void BinaryCacheStore::getFile(const std::string & path,
     Callback<std::shared_ptr<std::string>> callback) noexcept
 {
@@ -113,13 +123,71 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
         diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
 }
 
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
-    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+AutoCloseFD openFile(const Path & path)
 {
-    // FIXME: See if we can use the original source to reduce memory usage.
-    auto nar = make_ref<std::string>(narSource.drain());
+    auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    return fd;
+}
 
-    if (!repair && isValidPath(info.path)) return;
+struct FileSource : FdSource
+{
+    AutoCloseFD fd2;
+
+    FileSource(const Path & path)
+        : fd2(openFile(path))
+    {
+        fd = fd2.get();
+    }
+};
+
+ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
+    Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
+    std::function<ValidPathInfo(HashResult)> mkInfo)
+{
+    auto [fdTemp, fnTemp] = createTempFile();
+
+    AutoDelete autoDelete(fnTemp);
+
+    auto now1 = std::chrono::steady_clock::now();
+
+    /* Read the NAR simultaneously into a CompressionSink+FileSink (to
+       write the compressed NAR to disk), into a HashSink (to get the
+       NAR hash), and into a NarAccessor (to get the NAR listing). */
+    HashSink fileHashSink { htSHA256 };
+    std::shared_ptr<FSAccessor> narAccessor;
+    HashSink narHashSink { htSHA256 };
+    {
+    FdSink fileSink(fdTemp.get());
+    TeeSink teeSinkCompressed { fileSink, fileHashSink };
+    auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
+    TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
+    TeeSource teeSource { narSource, teeSinkUncompressed };
+    narAccessor = makeNarAccessor(teeSource);
+    compressionSink->finish();
+    fileSink.flush();
+    }
+
+    auto now2 = std::chrono::steady_clock::now();
+
+    auto info = mkInfo(narHashSink.finish());
+    auto narInfo = make_ref<NarInfo>(info);
+    narInfo->compression = compression;
+    auto [fileHash, fileSize] = fileHashSink.finish();
+    narInfo->fileHash = fileHash;
+    narInfo->fileSize = fileSize;
+    narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+        + (compression == "xz" ? ".xz" :
+           compression == "bzip2" ? ".bz2" :
+           compression == "br" ? ".br" :
+           "");
+
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
+        printStorePath(narInfo->path), info.narSize,
+        ((1.0 - (double) fileSize / info.narSize) * 100.0),
+        duration);
 
     /* Verify that all references are valid. This may do some .narinfo
        reads, but typically they'll already be cached. */
@@ -132,23 +200,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
                 printStorePath(info.path), printStorePath(ref));
         }
 
-    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
-
-    auto narInfo = make_ref<NarInfo>(info);
-
-    narInfo->narSize = nar->size();
-    narInfo->narHash = hashString(htSHA256, *nar);
-
-    if (info.narHash && info.narHash != narInfo->narHash)
-        throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
-
-    auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
-
-    auto narAccessor = makeNarAccessor(nar);
-
-    if (accessor_)
-        accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
-
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
@@ -160,33 +211,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 
             {
                 auto res = jsonRoot.placeholder("root");
-                listNar(res, narAccessor, "", true);
+                listNar(res, ref<FSAccessor>(narAccessor), "", true);
             }
         }
 
-        upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
+        upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json");
     }
 
-    /* Compress the NAR. */
-    narInfo->compression = compression;
-    auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar, parallelCompression);
-    auto now2 = std::chrono::steady_clock::now();
-    narInfo->fileHash = hashString(htSHA256, *narCompressed);
-    narInfo->fileSize = narCompressed->size();
-
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
-        printStorePath(narInfo->path), narInfo->narSize,
-        ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
-        duration);
-
-    narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
-        + (compression == "xz" ? ".xz" :
-           compression == "bzip2" ? ".bz2" :
-           compression == "br" ? ".br" :
-           "");
-
     /* Optionally maintain an index of DWARF debug info files
        consisting of JSON files named 'debuginfo/<build-id>' that
        specify the NAR file and member containing the debug info. */
@@ -247,12 +278,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
+        upsertFile(narInfo->url,
+            std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
+            "application/x-nix-nar");
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += nar->size();
-    stats.narWriteCompressedBytes += narCompressed->size();
+    stats.narWriteBytes += info.narSize;
+    stats.narWriteCompressedBytes += fileSize;
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
@@ -261,6 +294,41 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     writeNarInfo(narInfo);
 
     stats.narInfoWrite++;
+
+    return narInfo;
+}
+
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
+    RepairFlag repair, CheckSigsFlag checkSigs)
+{
+    if (!repair && isValidPath(info.path)) {
+        // FIXME: copyNAR -> null sink
+        narSource.drain();
+        return;
+    }
+
+    addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) {
+        /* FIXME reinstate these, once we can correctly do hash modulo sink as
+           needed. We need to throw here in case we uploaded a corrupted store path. */
+        // assert(info.narHash == nar.first);
+        // assert(info.narSize == nar.second);
+        return info;
+    }});
+}
+
+StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
+    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
+{
+    if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
+        unsupported("addToStoreFromDump");
+    return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
+        ValidPathInfo info {
+            makeFixedOutputPath(method, nar.first, name),
+            nar.first,
+        };
+        info.narSize = nar.second;
+        return info;
+    })->path;
 }
 
 bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
@@ -275,14 +343,10 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
-    uint64_t narSize = 0;
+    LengthSink narSize;
+    TeeSink tee { sink, narSize };
 
-    LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
-        sink(data, len);
-        narSize += len;
-    });
-
-    auto decompressor = makeDecompressionSink(info->compression, wrapperSink);
+    auto decompressor = makeDecompressionSink(info->compression, tee);
 
     try {
         getFile(info->url, *decompressor);
@@ -294,7 +358,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 
     stats.narRead++;
     //stats.narReadCompressedBytes += nar->size(); // FIXME
-    stats.narReadBytes += narSize;
+    stats.narReadBytes += narSize.length;
 }
 
 void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
@@ -332,44 +396,52 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
     FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
-    // FIXME: some cut&paste from LocalStore::addToStore().
+    /* FIXME: Make BinaryCacheStore::addToStoreCommon support
+       non-recursive+sha256 so we can just use the default
+       implementation of this method in terms of addToStoreFromDump. */
 
-    /* Read the whole path into memory. This is not a very scalable
-       method for very large paths, but `copyPath' is mainly used for
-       small files. */
-    StringSink sink;
-    Hash h;
+    HashSink sink { hashAlgo };
     if (method == FileIngestionMethod::Recursive) {
         dumpPath(srcPath, sink, filter);
-        h = hashString(hashAlgo, *sink.s);
     } else {
-        auto s = readFile(srcPath);
-        dumpString(s, sink);
-        h = hashString(hashAlgo, s);
+        readFile(srcPath, sink);
     }
+    auto h = sink.finish().first;
 
-    ValidPathInfo info(makeFixedOutputPath(method, h, name));
-
-    auto source = StringSource { *sink.s };
-    addToStore(info, source, repair, CheckSigs, nullptr);
-
-    return std::move(info.path);
+    auto source = sinkToSource([&](Sink & sink) {
+        dumpPath(srcPath, sink, filter);
+    });
+    return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
+        ValidPathInfo info {
+            makeFixedOutputPath(method, h, name),
+            nar.first,
+        };
+        info.narSize = nar.second;
+        info.ca = FixedOutputHash {
+            .method = method,
+            .hash = h,
+        };
+        return info;
+    })->path;
 }
 
 StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
-    ValidPathInfo info(computeStorePathForText(name, s, references));
-    info.references = references;
+    auto textHash = hashString(htSHA256, s);
+    auto path = makeTextPath(name, textHash, references);
 
-    if (repair || !isValidPath(info.path)) {
-        StringSink sink;
-        dumpString(s, sink);
-        auto source = StringSource { *sink.s };
-        addToStore(info, source, repair, CheckSigs, nullptr);
-    }
+    if (!repair && isValidPath(path))
+        return path;
 
-    return std::move(info.path);
+    auto source = StringSource { s };
+    return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
+        ValidPathInfo info { path, nar.first };
+        info.narSize = nar.second;
+        info.ca = TextHash { textHash };
+        info.references = references;
+        return info;
+    })->path;
 }
 
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()

src/libstore/binary-cache-store.hh

@@ -11,17 +11,21 @@ namespace nix {
 
 struct NarInfo;
 
-class BinaryCacheStore : public Store
+struct BinaryCacheStoreConfig : virtual StoreConfig
 {
-public:
+    using StoreConfig::StoreConfig;
 
-    const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
-    const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
-    const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
-    const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
-    const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
-    const Setting<bool> parallelCompression{this, false, "parallel-compression",
+    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
+    const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
+    const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
+    const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
+    const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
+    const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
         "enable multi-threading compression, available for xz only currently"};
+};
+
+class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
+{
 
 private:
 
@@ -36,9 +40,13 @@ public:
     virtual bool fileExists(const std::string & path) = 0;
 
     virtual void upsertFile(const std::string & path,
-        const std::string & data,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) = 0;
 
+    void upsertFile(const std::string & path,
+        std::string && data,
+        const std::string & mimeType);
+
     /* Note: subclasses must implement at least one of the two
        following getFile() methods. */
 
@@ -54,7 +62,7 @@ public:
 
 public:
 
-    virtual void init();
+    virtual void init() override;
 
 private:
 
@@ -64,6 +72,10 @@ private:
 
     void writeNarInfo(ref<NarInfo> narInfo);
 
+    ref<const ValidPathInfo> addToStoreCommon(
+        Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
+        std::function<ValidPathInfo(HashResult)> mkInfo);
+
 public:
 
     bool isValidPathUncached(const StorePath & path) override;
@@ -75,8 +87,10 @@ public:
     { unsupported("queryPathFromHashPart"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override;
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
+
+    StorePath addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method, HashType hashAlgo,

src/libstore/builtins/buildenv.hh

@@ -9,7 +9,7 @@ struct Package {
     Path path;
     bool active;
     int priority;
-    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
+    Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
 };
 
 typedef std::vector<Package> Packages;

src/libstore/builtins/fetchurl.cc

@@ -58,17 +58,14 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         }
     };
 
-    /* We always have one output, and if it's a fixed-output derivation (as
-       checked below) it must be the only output */
-    auto & output = drv.outputs.begin()->second;
-
     /* Try the hashed mirrors first. */
-    if (output.hash && output.hash->method == FileIngestionMethod::Flat)
+    if (getAttr("outputHashMode") == "flat")
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto & h = output.hash->hash;
-                fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
+                std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
+                Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
+                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {
                 debug(e.what());

src/libstore/cgroup.cc

@@ -4,7 +4,10 @@
 #include "util.hh"
 
 #include <chrono>
+#include <cmath>
+#include <regex>
 #include <unordered_set>
+#include <thread>
 
 #include <dirent.h>
 

src/libstore/content-address.cc

@@ -1,12 +1,16 @@
+#include "args.hh"
 #include "content-address.hh"
+#include "split.hh"
 
 namespace nix {
 
-std::string FixedOutputHash::printMethodAlgo() const {
-    return makeFileIngestionPrefix(method) + printHashType(*hash.type);
+std::string FixedOutputHash::printMethodAlgo() const
+{
+    return makeFileIngestionPrefix(method) + printHashType(hash.type);
 }
 
-std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
+std::string makeFileIngestionPrefix(const FileIngestionMethod m)
+{
     switch (m) {
     case FileIngestionMethod::Flat:
         return "";
@@ -24,11 +28,8 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
         + hash.to_string(Base32, true);
 }
 
-// FIXME Put this somewhere?
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
-std::string renderContentAddress(ContentAddress ca) {
+std::string renderContentAddress(ContentAddress ca)
+{
     return std::visit(overloaded {
         [](TextHash th) {
             return "text:" + th.hash.to_string(Base32, true);
@@ -39,47 +40,110 @@ std::string renderContentAddress(ContentAddress ca) {
     }, ca);
 }
 
-ContentAddress parseContentAddress(std::string_view rawCa) {
-    auto prefixSeparator = rawCa.find(':');
-    if (prefixSeparator != string::npos) {
-        auto prefix = string(rawCa, 0, prefixSeparator);
-        if (prefix == "text") {
-            auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
-            Hash hash = Hash(string(hashTypeAndHash));
-            if (*hash.type != htSHA256) {
-                throw Error("parseContentAddress: the text hash should have type SHA256");
-            }
-            return TextHash { hash };
-        } else if (prefix == "fixed") {
-            // This has to be an inverse of makeFixedOutputCA
-            auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
-            if (methodAndHash.substr(0,2) == "r:") {
-                std::string_view hashRaw = methodAndHash.substr(2,string::npos);
-                return FixedOutputHash {
-                    .method = FileIngestionMethod::Recursive,
-                    .hash = Hash(string(hashRaw)),
-                };
-            } else {
-                std::string_view hashRaw = methodAndHash;
-                return FixedOutputHash {
-                    .method = FileIngestionMethod::Flat,
-                    .hash = Hash(string(hashRaw)),
-                };
-            }
-        } else {
-            throw Error("parseContentAddress: format not recognized; has to be text or fixed");
+std::string renderContentAddressMethod(ContentAddressMethod cam)
+{
+    return std::visit(overloaded {
+        [](TextHashMethod &th) {
+            return std::string{"text:"} + printHashType(htSHA256);
+        },
+        [](FixedOutputHashMethod &fshm) {
+            return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
         }
-    } else {
-        throw Error("Not a content address because it lacks an appropriate prefix");
+    }, cam);
+}
+
+/*
+  Parses content address strings up to the hash.
+ */
+static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
+{
+    std::string_view wholeInput { rest };
+
+    std::string_view prefix;
+    {
+        auto optPrefix = splitPrefixTo(rest, ':');
+        if (!optPrefix)
+            throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", wholeInput);
+        prefix = *optPrefix;
     }
+
+    auto parseHashType_ = [&](){
+        auto hashTypeRaw = splitPrefixTo(rest, ':');
+        if (!hashTypeRaw)
+            throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
+        HashType hashType = parseHashType(*hashTypeRaw);
+        return std::move(hashType);
+    };
+
+    // Switch on prefix
+    if (prefix == "text") {
+        // No parsing of the ingestion method, "text" only support flat.
+        HashType hashType = parseHashType_();
+        if (hashType != htSHA256)
+            throw Error("text content address hash should use %s, but instead uses %s",
+                printHashType(htSHA256), printHashType(hashType));
+        return TextHashMethod {};
+    } else if (prefix == "fixed") {
+        // Parse method
+        auto method = FileIngestionMethod::Flat;
+        if (splitPrefix(rest, "r:"))
+            method = FileIngestionMethod::Recursive;
+        HashType hashType = parseHashType_();
+        return FixedOutputHashMethod {
+            .fileIngestionMethod = method,
+            .hashType = std::move(hashType),
+        };
+    } else
+        throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
+}
+
+ContentAddress parseContentAddress(std::string_view rawCa) {
+    auto rest = rawCa;
+
+    ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
+
+    return std::visit(
+        overloaded {
+            [&](TextHashMethod thm) {
+                return ContentAddress(TextHash {
+                    .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
+                });
+            },
+            [&](FixedOutputHashMethod fohMethod) {
+                return ContentAddress(FixedOutputHash {
+                    .method = fohMethod.fileIngestionMethod,
+                    .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
+                });
+            },
+        }, caMethod);
+}
+
+ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
+{
+    std::string_view asPrefix {std::string{caMethod} + ":"};
+    return parseContentAddressMethodPrefix(asPrefix);
+}
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt)
+{
+    return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt);
 };
 
-std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
-    return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
-};
-
-std::string renderContentAddress(std::optional<ContentAddress> ca) {
+std::string renderContentAddress(std::optional<ContentAddress> ca)
+{
     return ca ? renderContentAddress(*ca) : "";
 }
 
+Hash getContentAddressHash(const ContentAddress & ca)
+{
+    return std::visit(overloaded {
+        [](TextHash th) {
+            return th.hash;
+        },
+        [](FixedOutputHash fsh) {
+            return fsh.hash;
+        }
+    }, ca);
+}
+
 }

src/libstore/content-address.hh

@@ -53,4 +53,25 @@ ContentAddress parseContentAddress(std::string_view rawCa);
 
 std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
 
+Hash getContentAddressHash(const ContentAddress & ca);
+
+/*
+  We only have one way to hash text with references, so this is single-value
+  type is only useful in std::variant.
+*/
+struct TextHashMethod { };
+struct FixedOutputHashMethod {
+  FileIngestionMethod fileIngestionMethod;
+  HashType hashType;
+};
+
+typedef std::variant<
+    TextHashMethod,
+    FixedOutputHashMethod
+  > ContentAddressMethod;
+
+ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
+
+std::string renderContentAddressMethod(ContentAddressMethod caMethod);
+
 }

src/libstore/daemon.cc

@@ -2,7 +2,6 @@
 #include "monitor-fd.hh"
 #include "worker-protocol.hh"
 #include "store-api.hh"
-#include "local-store.hh"
 #include "finally.hh"
 #include "affinity.hh"
 #include "archive.hh"
@@ -81,12 +80,12 @@ struct TunnelLogger : public Logger
         showErrorInfo(oss, ei, false);
 
         StringSink buf;
-        buf << STDERR_NEXT << oss.str() << "\n";
+        buf << STDERR_NEXT << oss.str();
         enqueueMsg(*buf.s);
     }
 
     /* startWork() means that we're starting an operation for which we
-      want to send out stderr to the client. */
+       want to send out stderr to the client. */
     void startWork()
     {
         auto state(state_.lock());
@@ -102,17 +101,20 @@ struct TunnelLogger : public Logger
 
     /* stopWork() means that we're done; stop sending stderr to the
        client. */
-    void stopWork(bool success = true, const string & msg = "", unsigned int status = 0)
+    void stopWork(const Error * ex = nullptr)
     {
         auto state(state_.lock());
 
         state->canSendStderr = false;
 
-        if (success)
+        if (!ex)
             to << STDERR_LAST;
         else {
-            to << STDERR_ERROR << msg;
-            if (status != 0) to << status;
+            if (GET_PROTOCOL_MINOR(clientVersion) >= 26) {
+                to << STDERR_ERROR << *ex;
+            } else {
+                to << STDERR_ERROR << ex->what() << ex->status;
+            }
         }
     }
 
@@ -173,31 +175,6 @@ struct TunnelSource : BufferedSource
     }
 };
 
-/* If the NAR archive contains a single file at top-level, then save
-   the contents of the file to `s'.  Otherwise barf. */
-struct RetrieveRegularNARSink : ParseSink
-{
-    bool regular;
-    string s;
-
-    RetrieveRegularNARSink() : regular(true) { }
-
-    void createDirectory(const Path & path)
-    {
-        regular = false;
-    }
-
-    void receiveContents(unsigned char * data, unsigned int len)
-    {
-        s.append((const char *) data, len);
-    }
-
-    void createSymlink(const Path & path, const string & target)
-    {
-        regular = false;
-    }
-};
-
 struct ClientSettings
 {
     bool keepFailed;
@@ -257,7 +234,7 @@ struct ClientSettings
                 else if (setSubstituters(settings.extraSubstituters))
                     ;
                 else
-                    warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
+                    debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
             } catch (UsageError & e) {
                 warn(e.what());
             }
@@ -265,6 +242,23 @@ struct ClientSettings
     }
 };
 
+static void writeValidPathInfo(
+    ref<Store> store,
+    unsigned int clientVersion,
+    Sink & to,
+    std::shared_ptr<const ValidPathInfo> info)
+{
+    to << (info->deriver ? store->printStorePath(*info->deriver) : "")
+       << info->narHash.to_string(Base16, false);
+    worker_proto::write(*store, to, info->references);
+    to << info->registrationTime << info->narSize;
+    if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
+        to << info->ultimate
+           << info->sigs
+           << renderContentAddress(info->ca);
+    }
+}
+
 static void performOp(TunnelLogger * logger, ref<Store> store,
     TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
     Source & from, BufferedSink & to, unsigned int op)
@@ -281,11 +275,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQueryValidPaths: {
-        auto paths = readStorePaths<StorePathSet>(*store, from);
+        auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto res = store->queryValidPaths(paths);
         logger->stopWork();
-        writeStorePaths(*store, to, res);
+        worker_proto::write(*store, to, res);
         break;
     }
 
@@ -301,11 +295,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePaths: {
-        auto paths = readStorePaths<StorePathSet>(*store, from);
+        auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto res = store->querySubstitutablePaths(paths);
         logger->stopWork();
-        writeStorePaths(*store, to, res);
+        worker_proto::write(*store, to, res);
         break;
     }
 
@@ -334,7 +328,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         logger->stopWork();
-        writeStorePaths(*store, to, paths);
+        worker_proto::write(*store, to, paths);
         break;
     }
 
@@ -350,9 +344,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopQueryDerivationOutputMap: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
-        OutputPathMap outputs = store->queryDerivationOutputMap(path);
+        auto outputs = store->queryPartialDerivationOutputMap(path);
         logger->stopWork();
-        writeOutputPathMap(*store, to, outputs);
+        worker_proto::write(*store, to, outputs);
         break;
     }
 
@@ -375,52 +369,90 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopAddToStore: {
-        std::string s, baseName;
-        FileIngestionMethod method;
-        {
-            bool fixed; uint8_t recursive;
-            from >> baseName >> fixed /* obsolete */ >> recursive >> s;
-            if (recursive > (uint8_t) FileIngestionMethod::Recursive)
-                throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
-            method = FileIngestionMethod { recursive };
-            /* Compatibility hack. */
-            if (!fixed) {
-                s = "sha256";
-                method = FileIngestionMethod::Recursive;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
+            auto name = readString(from);
+            auto camStr = readString(from);
+            auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+            bool repairBool;
+            from >> repairBool;
+            auto repair = RepairFlag{repairBool};
+
+            logger->startWork();
+            auto pathInfo = [&]() {
+                // NB: FramedSource must be out of scope before logger->stopWork();
+                ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr);
+                FramedSource source(from);
+                // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
+                return std::visit(overloaded {
+                    [&](TextHashMethod &_) {
+                        // We could stream this by changing Store
+                        std::string contents = source.drain();
+                        auto path = store->addTextToStore(name, contents, refs, repair);
+                        return store->queryPathInfo(path);
+                    },
+                    [&](FixedOutputHashMethod &fohm) {
+                        if (!refs.empty())
+                            throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
+                        auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
+                        return store->queryPathInfo(path);
+                    },
+                }, contentAddressMethod);
+            }();
+            logger->stopWork();
+
+            to << store->printStorePath(pathInfo->path);
+            writeValidPathInfo(store, clientVersion, to, pathInfo);
+
+        } else {
+            HashType hashAlgo;
+            std::string baseName;
+            FileIngestionMethod method;
+            {
+                bool fixed;
+                uint8_t recursive;
+                std::string hashAlgoRaw;
+                from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
+                if (recursive > (uint8_t) FileIngestionMethod::Recursive)
+                    throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
+                method = FileIngestionMethod { recursive };
+                /* Compatibility hack. */
+                if (!fixed) {
+                    hashAlgoRaw = "sha256";
+                    method = FileIngestionMethod::Recursive;
+                }
+                hashAlgo = parseHashType(hashAlgoRaw);
             }
+
+            StringSink saved;
+            TeeSource savedNARSource(from, saved);
+            RetrieveRegularNARSink savedRegular { saved };
+
+            if (method == FileIngestionMethod::Recursive) {
+                /* Get the entire NAR dump from the client and save it to
+                  a string so that we can pass it to
+                  addToStoreFromDump(). */
+                ParseSink sink; /* null sink; just parse the NAR */
+                parseDump(sink, savedNARSource);
+            } else
+                parseDump(savedRegular, from);
+
+            logger->startWork();
+            if (!savedRegular.regular) throw Error("regular file expected");
+
+            // FIXME: try to stream directly from `from`.
+            StringSource dumpSource { *saved.s };
+            auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
+            logger->stopWork();
+
+            to << store->printStorePath(path);
         }
-        HashType hashAlgo = parseHashType(s);
-
-        TeeSource savedNAR(from);
-        RetrieveRegularNARSink savedRegular;
-
-        if (method == FileIngestionMethod::Recursive) {
-            /* Get the entire NAR dump from the client and save it to
-               a string so that we can pass it to
-               addToStoreFromDump(). */
-            ParseSink sink; /* null sink; just parse the NAR */
-            parseDump(sink, savedNAR);
-        } else
-            parseDump(savedRegular, from);
-
-        logger->startWork();
-        if (!savedRegular.regular) throw Error("regular file expected");
-
-        auto path = store->addToStoreFromDump(
-            method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
-            baseName,
-            method,
-            hashAlgo);
-        logger->stopWork();
-
-        to << store->printStorePath(path);
         break;
     }
 
     case wopAddTextToStore: {
         string suffix = readString(from);
         string s = readString(from);
-        auto refs = readStorePaths<StorePathSet>(*store, from);
+        auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         logger->startWork();
         auto path = store->addTextToStore(suffix, s, refs, NoRepair);
         logger->stopWork();
@@ -442,7 +474,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopImportPaths: {
         logger->startWork();
         TunnelSource source(from, to);
-        auto paths = store->importPaths(source, nullptr,
+        auto paths = store->importPaths(source,
             trusted ? NoCheckSigs : CheckSigs);
         logger->stopWork();
         Strings paths2;
@@ -474,11 +506,63 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopBuildDerivation: {
         auto drvPath = store->parseStorePath(readString(from));
         BasicDerivation drv;
-        readDerivation(from, *store, drv);
+        readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
         BuildMode buildMode = (BuildMode) readInt(from);
         logger->startWork();
-        if (!trusted)
-            throw Error("you are not privileged to build derivations");
+
+        /* Content-addressed derivations are trustless because their output paths
+           are verified by their content alone, so any derivation is free to
+           try to produce such a path.
+
+           Input-addressed derivation output paths, however, are calculated
+           from the derivation closure that produced them---even knowing the
+           root derivation is not enough. That the output data actually came
+           from those derivations is fundamentally unverifiable, but the daemon
+           trusts itself on that matter. The question instead is whether the
+           submitted plan has rights to the output paths it wants to fill, and
+           at least the derivation closure proves that.
+
+           It would have been nice if input-address algorithm merely depended
+           on the build time closure, rather than depending on the derivation
+           closure. That would mean input-addressed paths used at build time
+           would just be trusted and not need their own evidence. This is in
+           fact fine as the same guarantees would hold *inductively*: either
+           the remote builder has those paths and already trusts them, or it
+           needs to build them too and thus their evidence must be provided in
+           turn.  The advantage of this variant algorithm is that the evidence
+           for input-addressed paths which the remote builder already has
+           doesn't need to be sent again.
+
+           That said, now that we have floating CA derivations, it is better
+           that people just migrate to those which also solve this problem, and
+           others. It's the same migration difficulty with strictly more
+           benefit.
+
+           Lastly, do note that when we parse fixed-output content-addressed
+           derivations, we throw out the precomputed output paths and just
+           store the hashes, so there aren't two competing sources of truth an
+           attacker could exploit. */
+        if (drv.type() == DerivationType::InputAddressed && !trusted)
+            throw Error("you are not privileged to build input-addressed derivations");
+
+        /* Make sure that the non-input-addressed derivations that got this far
+           are in fact content-addressed if we don't trust them. */
+        assert(derivationIsCA(drv.type()) || trusted);
+
+        /* Recompute the derivation path when we cannot trust the original. */
+        if (!trusted) {
+            /* Recomputing the derivation path for input-address derivations
+               makes it harder to audit them after the fact, since we need the
+               original not-necessarily-resolved derivation to verify the drv
+               derivation as adequate claim to the input-addressed output
+               paths. */
+            assert(derivationIsCA(drv.type()));
+
+            Derivation drv2;
+            static_cast<BasicDerivation &>(drv2) = drv;
+            drvPath = writeDerivation(*store, Derivation { drv2 });
+        }
+
         auto res = store->buildDerivation(drvPath, drv, buildMode);
         logger->stopWork();
         to << res.status << res.errorMsg;
@@ -541,7 +625,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case wopCollectGarbage: {
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
-        options.pathsToDelete = readStorePaths<StorePathSet>(*store, from);
+        options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         from >> options.ignoreLiveness >> options.maxFreed;
         // obsolete fields
         readInt(from);
@@ -602,7 +686,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         SubstitutablePathInfos infos;
-        store->querySubstitutablePathInfos({path}, infos);
+        store->querySubstitutablePathInfos({{path, std::nullopt}}, infos);
         logger->stopWork();
         auto i = infos.find(path);
         if (i == infos.end())
@@ -610,7 +694,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            writeStorePaths(*store, to, i->second.references);
+            worker_proto::write(*store, to, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -618,16 +702,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     }
 
     case wopQuerySubstitutablePathInfos: {
-        auto paths = readStorePaths<StorePathSet>(*store, from);
-        logger->startWork();
         SubstitutablePathInfos infos;
-        store->querySubstitutablePathInfos(paths, infos);
+        StorePathCAMap pathsMap = {};
+        if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
+            auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+            for (auto & path : paths)
+                pathsMap.emplace(path, std::nullopt);
+        } else
+            pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
+        logger->startWork();
+        store->querySubstitutablePathInfos(pathsMap, infos);
         logger->stopWork();
         to << infos.size();
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            writeStorePaths(*store, to, i.second.references);
+            worker_proto::write(*store, to, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -637,7 +727,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto paths = store->queryAllValidPaths();
         logger->stopWork();
-        writeStorePaths(*store, to, paths);
+        worker_proto::write(*store, to, paths);
         break;
     }
 
@@ -654,15 +744,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (info) {
             if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
                 to << 1;
-            to << (info->deriver ? store->printStorePath(*info->deriver) : "")
-               << info->narHash.to_string(Base16, false);
-            writeStorePaths(*store, to, info->references);
-            to << info->registrationTime << info->narSize;
-            if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
-                to << info->ultimate
-                   << info->sigs
-                   << renderContentAddress(info->ca);
-            }
+            writeValidPathInfo(store, clientVersion, to, info);
         } else {
             assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
             to << 0;
@@ -705,18 +787,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto path = store->parseStorePath(readString(from));
         logger->startWork();
         logger->stopWork();
-        dumpPath(store->printStorePath(path), to);
+        dumpPath(store->toRealPath(path), to);
         break;
     }
 
     case wopAddToStoreNar: {
         bool repair, dontCheckSigs;
-        ValidPathInfo info(store->parseStorePath(readString(from)));
+        auto path = store->parseStorePath(readString(from));
         auto deriver = readString(from);
+        auto narHash = Hash::parseAny(readString(from), htSHA256);
+        ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
-        info.narHash = Hash(readString(from), htSHA256);
-        info.references = readStorePaths<StorePathSet>(*store, from);
+        info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = parseContentAddressOpt(readString(from));
@@ -726,24 +809,37 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (!trusted)
             info.ultimate = false;
 
-        std::string saved;
-        std::unique_ptr<Source> source;
-        if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
-            source = std::make_unique<TunnelSource>(from, to);
-        else {
-            TeeSink tee(from);
-            parseDump(tee, tee.source);
-            saved = std::move(*tee.source.data);
-            source = std::make_unique<StringSource>(saved);
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
+            logger->startWork();
+            {
+                FramedSource source(from);
+                store->addToStore(info, source, (RepairFlag) repair,
+                    dontCheckSigs ? NoCheckSigs : CheckSigs);
+            }
+            logger->stopWork();
         }
 
-        logger->startWork();
+        else {
+            std::unique_ptr<Source> source;
+            if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
+                source = std::make_unique<TunnelSource>(from, to);
+            else {
+                StringSink saved;
+                TeeSource tee { from, saved };
+                ParseSink ether;
+                parseDump(ether, tee);
+                source = std::make_unique<StringSource>(std::move(*saved.s));
+            }
 
-        // FIXME: race if addToStore doesn't read source?
-        store->addToStore(info, *source, (RepairFlag) repair,
-            dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
+            logger->startWork();
+
+            // FIXME: race if addToStore doesn't read source?
+            store->addToStore(info, *source, (RepairFlag) repair,
+                dontCheckSigs ? NoCheckSigs : CheckSigs);
+
+            logger->stopWork();
+        }
 
-        logger->stopWork();
         break;
     }
 
@@ -753,12 +849,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             targets.push_back(store->parsePathWithOutputs(s));
         logger->startWork();
         StorePathSet willBuild, willSubstitute, unknown;
-        unsigned long long downloadSize, narSize;
+        uint64_t downloadSize, narSize;
         store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
         logger->stopWork();
-        writeStorePaths(*store, to, willBuild);
-        writeStorePaths(*store, to, willSubstitute);
-        writeStorePaths(*store, to, unknown);
+        worker_proto::write(*store, to, willBuild);
+        worker_proto::write(*store, to, willSubstitute);
+        worker_proto::write(*store, to, unknown);
         to << downloadSize << narSize;
         break;
     }
@@ -774,8 +870,7 @@ void processConnection(
     FdSink & to,
     TrustedFlag trusted,
     RecursiveFlag recursive,
-    const std::string & userName,
-    uid_t userId)
+    std::function<void(Store &)> authHook)
 {
     auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr;
 
@@ -816,15 +911,7 @@ void processConnection(
 
         /* If we can't accept clientVersion, then throw an error
            *here* (not above). */
-
-#if 0
-        /* Prevent users from doing something very dangerous. */
-        if (geteuid() == 0 &&
-            querySetting("build-users-group", "") == "")
-            throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
-#endif
-
-        store->createUser(userName, userId);
+        authHook(*store);
 
         tunnelLogger->stopWork();
         to.flush();
@@ -851,10 +938,11 @@ void processConnection(
                    during addTextToStore() / importPath().  If that
                    happens, just send the error message and exit. */
                 bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr;
-                tunnelLogger->stopWork(false, e.msg(), e.status);
+                tunnelLogger->stopWork(&e);
                 if (!errorAllowed) throw;
             } catch (std::bad_alloc & e) {
-                tunnelLogger->stopWork(false, "Nix daemon out of memory", 1);
+                auto ex = Error("Nix daemon out of memory");
+                tunnelLogger->stopWork(&ex);
                 throw;
             }
 
@@ -863,8 +951,13 @@ void processConnection(
             assert(!tunnelLogger->state_.lock()->canSendStderr);
         };
 
+    } catch (Error & e) {
+        tunnelLogger->stopWork(&e);
+        to.flush();
+        return;
     } catch (std::exception & e) {
-        tunnelLogger->stopWork(false, e.what(), 1);
+        auto ex = Error(e.what());
+        tunnelLogger->stopWork(&ex);
         to.flush();
         return;
     }

src/libstore/daemon.hh

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "serialise.hh"
 #include "store-api.hh"
 
@@ -12,7 +14,10 @@ void processConnection(
     FdSink & to,
     TrustedFlag trusted,
     RecursiveFlag recursive,
-    const std::string & userName,
-    uid_t userId);
+    /* Arbitrary hook to check authorization / initialize user data / whatever
+       after the protocol has been negotiated. The idea is that this function
+       and everything it calls doesn't know about this stuff, and the
+       `nix-daemon` handles that instead. */
+    std::function<void(Store &)> authHook);
 
 }

src/libstore/derivations.cc

@@ -4,16 +4,61 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "fs-accessor.hh"
-#include "istringstream_nocopy.hh"
 
 namespace nix {
 
-const StorePath & BasicDerivation::findOutput(const string & id) const
+std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
 {
-    auto i = outputs.find(id);
-    if (i == outputs.end())
-        throw Error("derivation has no output '%s'", id);
-    return i->second.path;
+    return std::visit(overloaded {
+        [](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
+            return { doi.path };
+        },
+        [&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
+            return {
+                dof.path(store, drvName, outputName)
+            };
+        },
+        [](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
+            return std::nullopt;
+        },
+    }, output);
+}
+
+
+StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
+    return store.makeFixedOutputPath(
+        hash.method, hash.hash,
+        outputPathName(drvName, outputName));
+}
+
+
+bool derivationIsCA(DerivationType dt) {
+    switch (dt) {
+    case DerivationType::InputAddressed: return false;
+    case DerivationType::CAFixed: return true;
+    case DerivationType::CAFloating: return true;
+    };
+    // Since enums can have non-variant values, but making a `default:` would
+    // disable exhaustiveness warnings.
+    assert(false);
+}
+
+bool derivationIsFixed(DerivationType dt) {
+    switch (dt) {
+    case DerivationType::InputAddressed: return false;
+    case DerivationType::CAFixed: return true;
+    case DerivationType::CAFloating: return false;
+    };
+    assert(false);
+}
+
+bool derivationIsImpure(DerivationType dt) {
+    switch (dt) {
+    case DerivationType::InputAddressed: return false;
+    case DerivationType::CAFixed: return true;
+    case DerivationType::CAFloating: return false;
+    };
+    assert(false);
 }
 
 
@@ -23,8 +68,8 @@ bool BasicDerivation::isBuiltin() const
 }
 
 
-StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, std::string_view name, RepairFlag repair)
+StorePath writeDerivation(Store & store,
+    const Derivation & drv, RepairFlag repair, bool readOnly)
 {
     auto references = drv.inputSrcs;
     for (auto & i : drv.inputDrvs)
@@ -32,11 +77,11 @@ StorePath writeDerivation(ref<Store> store,
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
-    auto suffix = std::string(name) + drvExtension;
-    auto contents = drv.unparse(*store, false);
-    return settings.readOnlyMode
-        ? store->computeStorePathForText(suffix, contents, references)
-        : store->addTextToStore(suffix, contents, references, repair);
+    auto suffix = std::string(drv.name) + drvExtension;
+    auto contents = drv.unparse(store, false);
+    return readOnly || settings.readOnlyMode
+        ? store.computeStorePathForText(suffix, contents, references)
+        : store.addTextToStore(suffix, contents, references, repair);
 }
 
 
@@ -68,12 +113,15 @@ static string parseString(std::istream & str)
     return res;
 }
 
+static void validatePath(std::string_view s) {
+    if (s.size() == 0 || s[0] != '/')
+        throw FormatError("bad path '%1%' in derivation", s);
+}
 
 static Path parsePath(std::istream & str)
 {
-    string s = parseString(str);
-    if (s.size() == 0 || s[0] != '/')
-        throw FormatError("bad path '%1%' in derivation", s);
+    auto s = parseString(str);
+    validatePath(s);
     return s;
 }
 
@@ -101,38 +149,63 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
-static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
+static DerivationOutput parseDerivationOutput(const Store & store,
+    std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
 {
-    expect(str, ","); auto path = store.parseStorePath(parsePath(str));
-    expect(str, ","); auto hashAlgo = parseString(str);
-    expect(str, ","); const auto hash = parseString(str);
-    expect(str, ")");
-
-    std::optional<FixedOutputHash> fsh;
     if (hashAlgo != "") {
         auto method = FileIngestionMethod::Flat;
         if (string(hashAlgo, 0, 2) == "r:") {
             method = FileIngestionMethod::Recursive;
-            hashAlgo = string(hashAlgo, 2);
+            hashAlgo = hashAlgo.substr(2);
         }
-        const HashType hashType = parseHashType(hashAlgo);
-        fsh = FixedOutputHash {
-            .method = std::move(method),
-            .hash = Hash(hash, hashType),
+        const auto hashType = parseHashType(hashAlgo);
+        if (hash != "") {
+            validatePath(pathS);
+            return DerivationOutput {
+                .output = DerivationOutputCAFixed {
+                    .hash = FixedOutputHash {
+                        .method = std::move(method),
+                        .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
+                    },
+                },
+            };
+        } else {
+            settings.requireExperimentalFeature("ca-derivations");
+            assert(pathS == "");
+            return DerivationOutput {
+                .output = DerivationOutputCAFloating {
+                    .method = std::move(method),
+                    .hashType = std::move(hashType),
+                },
+            };
+        }
+    } else {
+        validatePath(pathS);
+        return DerivationOutput {
+            .output = DerivationOutputInputAddressed {
+                .path = store.parseStorePath(pathS),
+            }
         };
     }
+}
 
-    return DerivationOutput {
-        .path = std::move(path),
-        .hash = std::move(fsh),
-    };
+static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+{
+    expect(str, ","); const auto pathS = parseString(str);
+    expect(str, ","); const auto hashAlgo = parseString(str);
+    expect(str, ","); const auto hash = parseString(str);
+    expect(str, ")");
+
+    return parseDerivationOutput(store, pathS, hashAlgo, hash);
 }
 
 
-static Derivation parseDerivation(const Store & store, const string & s)
+Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
 {
     Derivation drv;
-    istringstream_nocopy str(s);
+    drv.name = name;
+
+    std::istringstream str(std::move(s));
     expect(str, "Derive([");
 
     /* Parse the list of outputs. */
@@ -175,34 +248,6 @@ static Derivation parseDerivation(const Store & store, const string & s)
 }
 
 
-Derivation readDerivation(const Store & store, const Path & drvPath)
-{
-    try {
-        return parseDerivation(store, readFile(drvPath));
-    } catch (FormatError & e) {
-        throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
-    }
-}
-
-
-Derivation Store::derivationFromPath(const StorePath & drvPath)
-{
-    ensurePath(drvPath);
-    return readDerivation(drvPath);
-}
-
-
-Derivation Store::readDerivation(const StorePath & drvPath)
-{
-    auto accessor = getFSAccessor();
-    try {
-        return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
-    } catch (FormatError & e) {
-        throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
-    }
-}
-
-
 static void printString(string & res, std::string_view s)
 {
     char buf[s.size() * 2 + 2];
@@ -264,10 +309,23 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     for (auto & i : outputs) {
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
-        s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
-        s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
-        s += ','; printUnquotedString(s,
-            i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
+        std::visit(overloaded {
+            [&](DerivationOutputInputAddressed doi) {
+                s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
+                s += ','; printUnquotedString(s, "");
+                s += ','; printUnquotedString(s, "");
+            },
+            [&](DerivationOutputCAFixed dof) {
+                s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
+                s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
+                s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
+            },
+            [&](DerivationOutputCAFloating dof) {
+                s += ','; printUnquotedString(s, "");
+                s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
+                s += ','; printUnquotedString(s, "");
+            },
+        }, i.second.output);
         s += ')';
     }
 
@@ -319,59 +377,144 @@ bool isDerivation(const string & fileName)
 }
 
 
-bool BasicDerivation::isFixedOutput() const
+std::string outputPathName(std::string_view drvName, std::string_view outputName) {
+    std::string res { drvName };
+    if (outputName != "out") {
+        res += "-";
+        res += outputName;
+    }
+    return res;
+}
+
+
+DerivationType BasicDerivation::type() const
 {
-    return outputs.size() == 1 &&
-        outputs.begin()->first == "out" &&
-        outputs.begin()->second.hash;
+    std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
+    std::optional<HashType> floatingHashType;
+    for (auto & i : outputs) {
+        std::visit(overloaded {
+            [&](DerivationOutputInputAddressed _) {
+               inputAddressedOutputs.insert(i.first);
+            },
+            [&](DerivationOutputCAFixed _) {
+                fixedCAOutputs.insert(i.first);
+            },
+            [&](DerivationOutputCAFloating dof) {
+                floatingCAOutputs.insert(i.first);
+                if (!floatingHashType) {
+                    floatingHashType = dof.hashType;
+                } else {
+                    if (*floatingHashType != dof.hashType)
+                        throw Error("All floating outputs must use the same hash type");
+                }
+            },
+        }, i.second.output);
+    }
+
+    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+        throw Error("Must have at least one output");
+    } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+        return DerivationType::InputAddressed;
+    } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+        if (fixedCAOutputs.size() > 1)
+            // FIXME: Experimental feature?
+            throw Error("Only one fixed output is allowed for now");
+        if (*fixedCAOutputs.begin() != "out")
+            throw Error("Single fixed output must be named \"out\"");
+        return DerivationType::CAFixed;
+    } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
+        return DerivationType::CAFloating;
+    } else {
+        throw Error("Can't mix derivation output types");
+    }
 }
 
 
 DrvHashes drvHashes;
 
+/* pathDerivationModulo and hashDerivationModulo are mutually recursive
+ */
 
-/* Returns the hash of a derivation modulo fixed-output
-   subderivations.  A fixed-output derivation is a derivation with one
-   output (`out') for which an expected hash and hash algorithm are
-   specified (using the `outputHash' and `outputHashAlgo'
-   attributes).  We don't want changes to such derivations to
-   propagate upwards through the dependency graph, changing output
-   paths everywhere.
+/* Look up the derivation by value and memoize the
+   `hashDerivationModulo` call.
+ */
+static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
+{
+    auto h = drvHashes.find(drvPath);
+    if (h == drvHashes.end()) {
+        assert(store.isValidPath(drvPath));
+        // Cache it
+        h = drvHashes.insert_or_assign(
+            drvPath,
+            hashDerivationModulo(
+                store,
+                store.readDerivation(drvPath),
+                false)).first;
+    }
+    return h->second;
+}
 
-   For instance, if we change the url in a call to the `fetchurl'
-   function, we do not want to rebuild everything depending on it
-   (after all, (the hash of) the file being downloaded is unchanged).
-   So the *output paths* should not change.  On the other hand, the
-   *derivation paths* should change to reflect the new dependency
-   graph.
+/* See the header for interface details. These are the implementation details.
 
-   That's what this function does: it returns a hash which is just the
-   hash of the derivation ATerm, except that any input derivation
-   paths have been replaced by the result of a recursive call to this
-   function, and that for fixed-output derivations we return a hash of
-   its output path. */
-Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
+   For fixed-output derivations, each hash in the map is not the
+   corresponding output's content hash, but a hash of that hash along
+   with other constant data. The key point is that the value is a pure
+   function of the output's contents, and there are no preimage attacks
+   either spoofing an output's contents for a derivation, or
+   spoofing a derivation for an output's contents.
+
+   For regular derivations, it looks up each subderivation from its hash
+   and recurs. If the subderivation is also regular, it simply
+   substitutes the derivation path with its hash. If the subderivation
+   is fixed-output, however, it takes each output hash and pretends it
+   is a derivation hash producing a single "out" output. This is so we
+   don't leak the provenance of fixed outputs, reducing pointless cache
+   misses as the build itself won't know this.
+ */
+DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
 {
     /* Return a fixed hash for fixed-output derivations. */
-    if (drv.isFixedOutput()) {
-        DerivationOutputs::const_iterator i = drv.outputs.begin();
-        return hashString(htSHA256, "fixed:out:"
-            + i->second.hash->printMethodAlgo() + ":"
-            + i->second.hash->hash.to_string(Base16, false) + ":"
-            + store.printStorePath(i->second.path));
+    switch (drv.type()) {
+    case DerivationType::CAFloating:
+        throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
+    case DerivationType::CAFixed: {
+        std::map<std::string, Hash> outputHashes;
+        for (const auto & i : drv.outputs) {
+            auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
+            auto hash = hashString(htSHA256, "fixed:out:"
+                + dof.hash.printMethodAlgo() + ":"
+                + dof.hash.hash.to_string(Base16, false) + ":"
+                + store.printStorePath(dof.path(store, drv.name, i.first)));
+            outputHashes.insert_or_assign(i.first, std::move(hash));
+        }
+        return outputHashes;
+    }
+    case DerivationType::InputAddressed:
+        break;
     }
 
     /* For other derivations, replace the inputs paths with recursive
-       calls to this function.*/
+       calls to this function. */
     std::map<std::string, StringSet> inputs2;
     for (auto & i : drv.inputDrvs) {
-        auto h = drvHashes.find(i.first);
-        if (h == drvHashes.end()) {
-            assert(store.isValidPath(i.first));
-            h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store,
-                store.readDerivation(i.first), false)).first;
-        }
-        inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
+        const auto & res = pathDerivationModulo(store, i.first);
+        std::visit(overloaded {
+            // Regular non-CA derivation, replace derivation
+            [&](Hash drvHash) {
+                inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
+            },
+            // CA derivation's output hashes
+            [&](CaOutputHashes outputHashes) {
+                std::set<std::string> justOut = { "out" };
+                for (auto & output : i.second) {
+                    /* Put each one in with a single "out" output.. */
+                    const auto h = outputHashes.at(output);
+                    inputs2.insert_or_assign(
+                        h.to_string(Base16, false),
+                        justOut);
+                }
+            },
+        }, res);
     }
 
     return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
@@ -392,38 +535,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
 }
 
 
-StorePathSet BasicDerivation::outputPaths() const
-{
-    StorePathSet paths;
-    for (auto & i : outputs)
-        paths.insert(i.second.path);
-    return paths;
-}
-
 static DerivationOutput readDerivationOutput(Source & in, const Store & store)
 {
-    auto path = store.parseStorePath(readString(in));
-    auto hashAlgo = readString(in);
-    auto hash = readString(in);
+    const auto pathS = readString(in);
+    const auto hashAlgo = readString(in);
+    const auto hash = readString(in);
 
-    std::optional<FixedOutputHash> fsh;
-    if (hashAlgo != "") {
-        auto method = FileIngestionMethod::Flat;
-        if (string(hashAlgo, 0, 2) == "r:") {
-            method = FileIngestionMethod::Recursive;
-            hashAlgo = string(hashAlgo, 2);
-        }
-        auto hashType = parseHashType(hashAlgo);
-        fsh = FixedOutputHash {
-            .method = std::move(method),
-            .hash = Hash(hash, hashType),
-        };
-    }
-
-    return DerivationOutput {
-        .path = std::move(path),
-        .hash = std::move(fsh),
-    };
+    return parseDerivationOutput(store, pathS, hashAlgo, hash);
 }
 
 StringSet BasicDerivation::outputNames() const
@@ -434,9 +552,30 @@ StringSet BasicDerivation::outputNames() const
     return names;
 }
 
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
+    DerivationOutputsAndOptPaths outsAndOptPaths;
+    for (auto output : outputs)
+        outsAndOptPaths.insert(std::make_pair(
+            output.first,
+            std::make_pair(output.second, output.second.path(store, name, output.first))
+            )
+        );
+    return outsAndOptPaths;
+}
 
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
+std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
+    auto nameWithSuffix = drvPath.name();
+    constexpr std::string_view extension = ".drv";
+    assert(hasSuffix(nameWithSuffix, extension));
+    nameWithSuffix.remove_suffix(extension.size());
+    return nameWithSuffix;
+}
+
+
+Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
 {
+    drv.name = name;
+
     drv.outputs.clear();
     auto nr = readNum<size_t>(in);
     for (size_t n = 0; n < nr; n++) {
@@ -445,7 +584,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
+    drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -464,16 +603,26 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
 {
     out << drv.outputs.size();
     for (auto & i : drv.outputs) {
-        out << i.first
-            << store.printStorePath(i.second.path);
-        if (i.second.hash) {
-            out << i.second.hash->printMethodAlgo()
-                << i.second.hash->hash.to_string(Base16, false);
-        } else {
-            out << "" << "";
-        }
+        out << i.first;
+        std::visit(overloaded {
+            [&](DerivationOutputInputAddressed doi) {
+                out << store.printStorePath(doi.path)
+                    << ""
+                    << "";
+            },
+            [&](DerivationOutputCAFixed dof) {
+                out << store.printStorePath(dof.path(store, drv.name, i.first))
+                    << dof.hash.printMethodAlgo()
+                    << dof.hash.hash.to_string(Base16, false);
+            },
+            [&](DerivationOutputCAFloating dof) {
+                out << ""
+                    << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
+                    << "";
+            },
+        }, i.second.output);
     }
-    writeStorePaths(store, out, drv.inputSrcs);
+    worker_proto::write(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
     for (auto & i : drv.env)
@@ -487,5 +636,65 @@ std::string hashPlaceholder(const std::string & outputName)
     return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
 }
 
+std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
+{
+    auto drvNameWithExtension = drvPath.name();
+    auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
+    auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
+    return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
+}
+
+
+// N.B. Outputs are left unchanged
+static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
+
+    debug("Rewriting the derivation");
+
+    for (auto &rewrite: rewrites) {
+        debug("rewriting %s as %s", rewrite.first, rewrite.second);
+    }
+
+    drv.builder = rewriteStrings(drv.builder, rewrites);
+    for (auto & arg: drv.args) {
+        arg = rewriteStrings(arg, rewrites);
+    }
+
+    StringPairs newEnv;
+    for (auto & envVar: drv.env) {
+        auto envName = rewriteStrings(envVar.first, rewrites);
+        auto envValue = rewriteStrings(envVar.second, rewrites);
+        newEnv.emplace(envName, envValue);
+    }
+    drv.env = newEnv;
+}
+
+
+Sync<DrvPathResolutions> drvPathResolutions;
+
+std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
+    BasicDerivation resolved { *this };
+
+    // Input paths that we'll want to rewrite in the derivation
+    StringMap inputRewrites;
+
+    for (auto & input : inputDrvs) {
+        auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first);
+        StringSet newOutputNames;
+        for (auto & outputName : input.second) {
+            auto actualPathOpt = inputDrvOutputs.at(outputName);
+            if (!actualPathOpt)
+                return std::nullopt;
+            auto actualPath = *actualPathOpt;
+            inputRewrites.emplace(
+                downstreamPlaceholder(store, input.first, outputName),
+                store.printStorePath(actualPath));
+            resolved.inputSrcs.insert(std::move(actualPath));
+        }
+    }
+
+    rewriteDerivation(store, resolved, inputRewrites);
+
+    return resolved;
+}
 
 }

src/libstore/derivations.hh

@@ -4,8 +4,10 @@
 #include "types.hh"
 #include "hash.hh"
 #include "content-address.hh"
+#include "sync.hh"
 
 #include <map>
+#include <variant>
 
 
 namespace nix {
@@ -13,20 +15,82 @@ namespace nix {
 
 /* Abstract syntax of derivations. */
 
+/* The traditional non-fixed-output derivation type. */
+struct DerivationOutputInputAddressed
+{
+    /* Will need to become `std::optional<StorePath>` once input-addressed
+       derivations are allowed to depend on cont-addressed derivations */
+    StorePath path;
+};
+
+/* Fixed-output derivations, whose output paths are content addressed
+   according to that fixed output. */
+struct DerivationOutputCAFixed
+{
+    FixedOutputHash hash; /* hash used for expected hash computation */
+    StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+};
+
+/* Floating-output derivations, whose output paths are content addressed, but
+   not fixed, and so are dynamically calculated from whatever the output ends
+   up being. */
+struct DerivationOutputCAFloating
+{
+    /* information used for expected hash computation */
+    FileIngestionMethod method;
+    HashType hashType;
+};
+
 struct DerivationOutput
 {
-    StorePath path;
-    std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */
+    std::variant<
+        DerivationOutputInputAddressed,
+        DerivationOutputCAFixed,
+        DerivationOutputCAFloating
+    > output;
+    std::optional<HashType> hashAlgoOpt(const Store & store) const;
+    /* Note, when you use this function you should make sure that you're passing
+       the right derivation name. When in doubt, you should use the safer
+       interface provided by BasicDerivation::outputsAndOptPaths */
+    std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
 
+/* These are analogues to the previous DerivationOutputs data type, but they
+   also contains, for each output, the (optional) store path in which it would
+   be written. To calculate values of these types, see the corresponding
+   functions in BasicDerivation */
+typedef std::map<string, std::pair<DerivationOutput, StorePath>>
+  DerivationOutputsAndPaths;
+typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>>
+  DerivationOutputsAndOptPaths;
+
 /* For inputs that are sub-derivations, we specify exactly which
    output IDs we are interested in. */
 typedef std::map<StorePath, StringSet> DerivationInputs;
 
 typedef std::map<string, string> StringPairs;
 
+enum struct DerivationType : uint8_t {
+    InputAddressed,
+    CAFixed,
+    CAFloating,
+};
+
+/* Do the outputs of the derivation have paths calculated from their content,
+   or from the derivation itself? */
+bool derivationIsCA(DerivationType);
+
+/* Is the content of the outputs fixed a-priori via a hash? Never true for
+   non-CA derivations. */
+bool derivationIsFixed(DerivationType);
+
+/* Is the derivation impure and needs to access non-deterministic resources, or
+   pure and can be sandboxed? Note that whether or not we actually sandbox the
+   derivation is controlled separately. Never true for non-CA derivations. */
+bool derivationIsImpure(DerivationType);
+
 struct BasicDerivation
 {
     DerivationOutputs outputs; /* keyed on symbolic IDs */
@@ -35,24 +99,25 @@ struct BasicDerivation
     Path builder;
     Strings args;
     StringPairs env;
+    std::string name;
 
-    BasicDerivation() { }
+    BasicDerivation() = default;
     virtual ~BasicDerivation() { };
 
-    /* Return the path corresponding to the output identifier `id' in
-       the given derivation. */
-    const StorePath & findOutput(const std::string & id) const;
-
     bool isBuiltin() const;
 
     /* Return true iff this is a fixed-output derivation. */
-    bool isFixedOutput() const;
-
-    /* Return the output paths of a derivation. */
-    StorePathSet outputPaths() const;
+    DerivationType type() const;
 
     /* Return the output names of a derivation. */
     StringSet outputNames() const;
+
+    /* Calculates the maps that contains all the DerivationOutputs, but
+       augmented with knowledge of the Store paths they would be written
+       into. */
+    DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
+
+    static std::string_view nameFromPath(const StorePath & storePath);
 };
 
 struct Derivation : BasicDerivation
@@ -63,7 +128,17 @@ struct Derivation : BasicDerivation
     std::string unparse(const Store & store, bool maskOutputs,
         std::map<std::string, StringSet> * actualInputs = nullptr) const;
 
-    Derivation() { }
+    /* Return the underlying basic derivation but with these changes:
+
+	   1. Input drvs are emptied, but the outputs of them that were used are
+	      added directly to input sources.
+
+       2. Input placeholders are replaced with realized input store paths. */
+    std::optional<BasicDerivation> tryResolve(Store & store);
+
+    Derivation() = default;
+    Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
+    Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
 };
 
 
@@ -72,30 +147,96 @@ class Store;
 enum RepairFlag : bool { NoRepair = false, Repair = true };
 
 /* Write a derivation to the Nix store, and return its path. */
-StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
+StorePath writeDerivation(Store & store,
+    const Derivation & drv,
+    RepairFlag repair = NoRepair,
+    bool readOnly = false);
 
 /* Read a derivation from a file. */
-Derivation readDerivation(const Store & store, const Path & drvPath);
+Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
 
 // FIXME: remove
 bool isDerivation(const string & fileName);
 
-Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
+/* Calculate the name that will be used for the store path for this
+   output.
+
+   This is usually <drv-name>-<output-name>, but is just <drv-name> when
+   the output name is "out". */
+std::string outputPathName(std::string_view drvName, std::string_view outputName);
+
+// known CA drv's output hashes, current just for fixed-output derivations
+// whose output hashes are always known since they are fixed up-front.
+typedef std::map<std::string, Hash> CaOutputHashes;
+
+typedef std::variant<
+    Hash, // regular DRV normalized hash
+    CaOutputHashes
+> DrvHashModulo;
+
+/* Returns hashes with the details of fixed-output subderivations
+   expunged.
+
+   A fixed-output derivation is a derivation whose outputs have a
+   specified content hash and hash algorithm. (Currently they must have
+   exactly one output (`out'), which is specified using the `outputHash'
+   and `outputHashAlgo' attributes, but the algorithm doesn't assume
+   this.) We don't want changes to such derivations to propagate upwards
+   through the dependency graph, changing output paths everywhere.
+
+   For instance, if we change the url in a call to the `fetchurl'
+   function, we do not want to rebuild everything depending on it---after
+   all, (the hash of) the file being downloaded is unchanged.  So the
+   *output paths* should not change. On the other hand, the *derivation
+   paths* should change to reflect the new dependency graph.
+
+   For fixed-output derivations, this returns a map from the name of
+   each output to its hash, unique up to the output's contents.
+
+   For regular derivations, it returns a single hash of the derivation
+   ATerm, after subderivations have been likewise expunged from that
+   derivation.
+ */
+DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
 
 /* Memoisation of hashDerivationModulo(). */
-typedef std::map<StorePath, Hash> DrvHashes;
+typedef std::map<StorePath, DrvHashModulo> DrvHashes;
 
 extern DrvHashes drvHashes; // FIXME: global, not thread-safe
 
+/* Memoisation of `readDerivation(..).resove()`. */
+typedef std::map<
+    StorePath,
+    std::optional<StorePath>
+> DrvPathResolutions;
+
+// FIXME: global, though at least thread-safe.
+// FIXME: arguably overlaps with hashDerivationModulo memo table.
+extern Sync<DrvPathResolutions> drvPathResolutions;
+
 bool wantOutput(const string & output, const std::set<string> & wanted);
 
 struct Source;
 struct Sink;
 
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv);
+Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
 void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
 
+/* This creates an opaque and almost certainly unique string
+   deterministically from the output name.
+
+   It is used as a placeholder to allow derivations to refer to their
+   own outputs without needing to use the hash of a derivation in
+   itself, making the hash near-impossible to calculate. */
 std::string hashPlaceholder(const std::string & outputName);
 
+/* This creates an opaque and almost certainly unique string
+   deterministically from a derivation path and output name.
+
+   It is used as a placeholder to allow derivations to refer to
+   content-addressed paths whose content --- and thus the path
+   themselves --- isn't yet known. This occurs when a derivation has a
+   dependency which is a CA derivation. */
+std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
+
 }

src/libstore/dummy-store.cc

@@ -0,0 +1,67 @@
+#include "store-api.hh"
+#include "callback.hh"
+
+namespace nix {
+
+struct DummyStoreConfig : virtual StoreConfig {
+    using StoreConfig::StoreConfig;
+
+    const std::string name() override { return "Dummy Store"; }
+};
+
+struct DummyStore : public Store, public virtual DummyStoreConfig
+{
+    DummyStore(const std::string scheme, const std::string uri, const Params & params)
+        : DummyStore(params)
+    { }
+
+    DummyStore(const Params & params)
+        : StoreConfig(params)
+        , Store(params)
+    { }
+
+    string getUri() override
+    {
+        return *uriSchemes().begin();
+    }
+
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
+    {
+        callback(nullptr);
+    }
+
+    static std::set<std::string> uriSchemes() {
+        return {"dummy"};
+    }
+
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
+    { unsupported("queryPathFromHashPart"); }
+
+    void addToStore(const ValidPathInfo & info, Source & source,
+        RepairFlag repair, CheckSigsFlag checkSigs) override
+    { unsupported("addToStore"); }
+
+    StorePath addToStore(const string & name, const Path & srcPath,
+        FileIngestionMethod method, HashType hashAlgo,
+        PathFilter & filter, RepairFlag repair) override
+    { unsupported("addToStore"); }
+
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair) override
+    { unsupported("addTextToStore"); }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    { unsupported("narFromPath"); }
+
+    void ensurePath(const StorePath & path) override
+    { unsupported("ensurePath"); }
+
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode) override
+    { unsupported("buildDerivation"); }
+};
+
+static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;
+
+}

src/libstore/export-import.cc

@@ -7,24 +7,6 @@
 
 namespace nix {
 
-struct HashAndWriteSink : Sink
-{
-    Sink & writeSink;
-    HashSink hashSink;
-    HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
-    {
-    }
-    virtual void operator () (const unsigned char * data, size_t len)
-    {
-        writeSink(data, len);
-        hashSink(data, len);
-    }
-    Hash currentHash()
-    {
-        return hashSink.currentHash().first;
-    }
-};
-
 void Store::exportPaths(const StorePathSet & paths, Sink & sink)
 {
     auto sorted = topoSortPaths(paths);
@@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink)
 {
     auto info = queryPathInfo(path);
 
-    HashAndWriteSink hashAndWriteSink(sink);
+    HashSink hashSink(htSHA256);
+    TeeSink teeSink(sink, hashSink);
 
-    narFromPath(path, hashAndWriteSink);
+    narFromPath(path, teeSink);
 
     /* Refuse to export paths that have changed.  This prevents
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
-    Hash hash = hashAndWriteSink.currentHash();
-    if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
+    Hash hash = hashSink.currentHash().first;
+    if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
             printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
 
-    hashAndWriteSink
+    teeSink
         << exportMagic
         << printStorePath(path);
-    writeStorePaths(*this, hashAndWriteSink, info->references);
-    hashAndWriteSink
+    worker_proto::write(*this, teeSink, info->references);
+    teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
 }
 
-StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
+StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 {
     StorePaths res;
     while (true) {
@@ -77,33 +60,36 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
         if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
 
         /* Extract the NAR from the source. */
-        TeeSink tee(source);
-        parseDump(tee, tee.source);
+        StringSink saved;
+        TeeSource tee { source, saved };
+        ParseSink ether;
+        parseDump(ether, tee);
 
         uint32_t magic = readInt(source);
         if (magic != exportMagic)
             throw Error("Nix archive cannot be imported; wrong format");
 
-        ValidPathInfo info(parseStorePath(readString(source)));
+        auto path = parseStorePath(readString(source));
 
         //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
 
-        info.references = readStorePaths<StorePathSet>(*this, source);
-
+        auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
         auto deriver = readString(source);
+        auto narHash = hashString(htSHA256, *saved.s);
+
+        ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = parseStorePath(deriver);
-
-        info.narHash = hashString(htSHA256, *tee.source.data);
-        info.narSize = tee.source.data->size();
+        info.references = references;
+        info.narSize = saved.s->size();
 
         // Ignore optional legacy signature.
         if (readInt(source) == 1)
             readString(source);
 
         // Can't use underlying source, which would have been exhausted
-        auto source = StringSource { *tee.source.data };
-        addToStore(info, source, NoRepair, checkSigs, accessor);
+        auto source = StringSource { *saved.s };
+        addToStore(info, source, NoRepair, checkSigs);
 
         res.push_back(info.path);
     }

src/libstore/filetransfer.cc

@@ -5,6 +5,7 @@
 #include "s3.hh"
 #include "compression.hh"
 #include "finally.hh"
+#include "callback.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -22,6 +23,7 @@
 #include <queue>
 #include <random>
 #include <thread>
+#include <regex>
 
 using namespace std::string_literals;
 
@@ -29,7 +31,7 @@ namespace nix {
 
 FileTransferSettings fileTransferSettings;
 
-static GlobalConfig::Register r1(&fileTransferSettings);
+static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
 
 std::string resolveUri(const std::string & uri)
 {
@@ -56,7 +58,7 @@ struct curlFileTransfer : public FileTransfer
         Callback<FileTransferResult> callback;
         CURL * req = 0;
         bool active = false; // whether the handle has been added to the multi object
-        std::string status;
+        std::string statusMsg;
 
         unsigned int attempt = 0;
 
@@ -111,6 +113,9 @@ struct curlFileTransfer : public FileTransfer
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
             if (!request.mimeType.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
+            for (auto it = request.headers.begin(); it != request.headers.end(); ++it){
+                requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str());
+            }
         }
 
         ~TransferItem()
@@ -123,7 +128,7 @@ struct curlFileTransfer : public FileTransfer
             if (requestHeaders) curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
-                    fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
+                    fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri));
             } catch (...) {
                 ignoreException();
             }
@@ -144,6 +149,7 @@ struct curlFileTransfer : public FileTransfer
 
         LambdaSink finalSink;
         std::shared_ptr<CompressionSink> decompressionSink;
+        std::optional<StringSink> errorSink;
 
         std::exception_ptr writeException;
 
@@ -153,9 +159,19 @@ struct curlFileTransfer : public FileTransfer
                 size_t realSize = size * nmemb;
                 result.bodySize += realSize;
 
-                if (!decompressionSink)
+                if (!decompressionSink) {
                     decompressionSink = makeDecompressionSink(encoding, finalSink);
+                    if (! successfulStatuses.count(getHTTPStatus())) {
+                        // In this case we want to construct a TeeSink, to keep
+                        // the response around (which we figure won't be big
+                        // like an actual download should be) to improve error
+                        // messages.
+                        errorSink = StringSink { };
+                    }
+                }
 
+                if (errorSink)
+                    (*errorSink)((unsigned char *) contents, realSize);
                 (*decompressionSink)((unsigned char *) contents, realSize);
 
                 return realSize;
@@ -175,12 +191,13 @@ struct curlFileTransfer : public FileTransfer
             size_t realSize = size * nmemb;
             std::string line((char *) contents, realSize);
             printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line));
-            if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
+            static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase);
+            std::smatch match;
+            if (std::regex_match(line, match, statusLine)) {
                 result.etag = "";
-                auto ss = tokenizeString<vector<string>>(line, " ");
-                status = ss.size() >= 2 ? ss[1] : "";
                 result.data = std::make_shared<std::string>();
                 result.bodySize = 0;
+                statusMsg = trim(match[1]);
                 acceptRanges = false;
                 encoding = "";
             } else {
@@ -194,7 +211,9 @@ struct curlFileTransfer : public FileTransfer
                            the expected ETag on a 200 response, then shut
                            down the connection because we already have the
                            data. */
-                        if (result.etag == request.expectedETag && status == "200") {
+                        long httpStatus = 0;
+                        curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+                        if (result.etag == request.expectedETag && httpStatus == 200) {
                             debug(format("shutting down on 200 HTTP response with expected ETag"));
                             return 0;
                         }
@@ -408,16 +427,21 @@ struct curlFileTransfer : public FileTransfer
 
                 attempt++;
 
+                std::shared_ptr<std::string> response;
+                if (errorSink)
+                    response = errorSink->s;
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
+                    ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
                     : httpStatus != 0
                     ? FileTransferError(err,
-                        fmt("unable to %s '%s': HTTP error %d",
-                            request.verb(), request.uri, httpStatus)
+                        response,
+                        fmt("unable to %s '%s': HTTP error %d ('%s')",
+                            request.verb(), request.uri, httpStatus, statusMsg)
                         + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
                         )
                     : FileTransferError(err,
+                        response,
                         fmt("unable to %s '%s': %s (%d)",
                             request.verb(), request.uri, curl_easy_strerror(code), code));
 
@@ -675,7 +699,7 @@ struct curlFileTransfer : public FileTransfer
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 FileTransferResult res;
                 if (!s3Res.data)
-                    throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
+                    throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
                 res.data = s3Res.data;
                 callback(std::move(res));
 #else
@@ -820,6 +844,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
     }
 }
 
+template<typename... Args>
+FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
+    : Error(args...), error(error), response(response)
+{
+    const auto hf = hintfmt(args...);
+    // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
+    // to print different messages for different verbosity levels. For now
+    // we add some heuristics for detecting when we want to show the response.
+    if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) {
+            err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
+    } else {
+        err.hint = hf;
+    }
+}
+
 bool isUri(const string & s)
 {
     if (s.compare(0, 8, "channel:") == 0) return true;

src/libstore/filetransfer.hh

@@ -17,15 +17,30 @@ struct FileTransferSettings : Config
     Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
         "String appended to the user agent in HTTP requests."};
 
-    Setting<size_t> httpConnections{this, 25, "http-connections",
-        "Number of parallel HTTP connections.",
+    Setting<size_t> httpConnections{
+        this, 25, "http-connections",
+        R"(
+          The maximum number of parallel TCP connections used to fetch
+          files from binary caches and by other downloads. It defaults
+          to 25. 0 means no limit.
+        )",
         {"binary-caches-parallel-connections"}};
 
-    Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
-        "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
+    Setting<unsigned long> connectTimeout{
+        this, 0, "connect-timeout",
+        R"(
+          The timeout (in seconds) for establishing connections in the
+          binary cache substituter. It corresponds to `curl`’s
+          `--connect-timeout` option.
+        )"};
 
-    Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout",
-        "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."};
+    Setting<unsigned long> stalledDownloadTimeout{
+        this, 300, "stalled-download-timeout",
+        R"(
+          The timeout (in seconds) for receiving data from servers
+          during download. Nix cancels idle downloads after this
+          timeout's duration.
+        )"};
 
     Setting<unsigned int> tries{this, 5, "download-attempts",
         "How often Nix will attempt to download a file before giving up."};
@@ -36,6 +51,7 @@ extern FileTransferSettings fileTransferSettings;
 struct FileTransferRequest
 {
     std::string uri;
+    Headers headers;
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
@@ -103,10 +119,12 @@ class FileTransferError : public Error
 {
 public:
     FileTransfer::Error error;
+    std::shared_ptr<string> response; // intentionally optional
+
     template<typename... Args>
-    FileTransferError(FileTransfer::Error error, const Args & ... args)
-        : Error(args...), error(error)
-    { }
+    FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
+
+    virtual const char* sname() const override { return "FileTransferError"; }
 };
 
 bool isUri(const string & s);

src/libstore/gc.cc

@@ -85,8 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path)
 }
 
 
-Path LocalFSStore::addPermRoot(const StorePath & storePath,
-    const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
+Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)
 {
     Path gcRoot(canonPath(_gcRoot));
 
@@ -95,47 +94,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath,
                 "creating a garbage collector root (%1%) in the Nix store is forbidden "
                 "(are you running nix-build inside the store?)", gcRoot);
 
-    if (indirect) {
-        /* Don't clobber the link if it already exists and doesn't
-           point to the Nix store. */
-        if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
-            throw Error("cannot create symlink '%1%'; already exists", gcRoot);
-        makeSymlink(gcRoot, printStorePath(storePath));
-        addIndirectRoot(gcRoot);
-    }
-
-    else {
-        if (!allowOutsideRootsDir) {
-            Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str());
-
-            if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
-                throw Error(
-                    "path '%1%' is not a valid garbage collector root; "
-                    "it's not in the directory '%2%'",
-                    gcRoot, rootsDir);
-        }
-
-        if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
-            writeFile(gcRoot, "");
-        else
-            makeSymlink(gcRoot, printStorePath(storePath));
-    }
-
-    /* Check that the root can be found by the garbage collector.
-       !!! This can be very slow on machines that have many roots.
-       Instead of reading all the roots, it would be more efficient to
-       check if the root is in a directory in or linked from the
-       gcroots directory. */
-    if (settings.checkRootReachability) {
-        auto roots = findRoots(false);
-        if (roots[storePath].count(gcRoot) == 0)
-            logWarning({
-                .name = "GC root",
-                .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; "
-                    "therefore, '%2%' might be removed by the garbage collector",
-                    gcRoot, printStorePath(storePath))
-            });
-    }
+    /* Don't clobber the link if it already exists and doesn't
+       point to the Nix store. */
+    if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
+        throw Error("cannot create symlink '%1%'; already exists", gcRoot);
+    makeSymlink(gcRoot, printStorePath(storePath));
+    addIndirectRoot(gcRoot);
 
     /* Grab the global GC root, causing us to block while a GC is in
        progress.  This prevents the set of permanent roots from
@@ -262,11 +226,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
 void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 {
     auto foundRoot = [&](const Path & path, const Path & target) {
-        auto storePath = maybeParseStorePath(toStorePath(target));
-        if (storePath && isValidPath(*storePath))
-            roots[std::move(*storePath)].emplace(path);
-        else
-            printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
+        try {
+            auto storePath = toStorePath(target).first;
+            if (isValidPath(storePath))
+                roots[std::move(storePath)].emplace(path);
+            else
+                printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
+        } catch (BadStorePath &) { }
     };
 
     try {
@@ -472,15 +438,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 
     for (auto & [target, links] : unchecked) {
         if (!isInStore(target)) continue;
-        Path pathS = toStorePath(target);
-        if (!isStorePath(pathS)) continue;
-        auto path = parseStorePath(pathS);
-        if (!isValidPath(path)) continue;
-        debug("got additional root '%1%'", pathS);
-        if (censor)
-            roots[path].insert(censored);
-        else
-            roots[path].insert(links.begin(), links.end());
+        try {
+            auto path = toStorePath(target).first;
+            if (!isValidPath(path)) continue;
+            debug("got additional root '%1%'", printStorePath(path));
+            if (censor)
+                roots[path].insert(censored);
+            else
+                roots[path].insert(links.begin(), links.end());
+        } catch (BadStorePath &) { }
     }
 }
 
@@ -498,7 +464,7 @@ struct LocalStore::GCState
     StorePathSet alive;
     bool gcKeepOutputs;
     bool gcKeepDerivations;
-    unsigned long long bytesInvalidated;
+    uint64_t bytesInvalidated;
     bool moveToTrash = true;
     bool shouldDelete;
     GCState(const GCOptions & options, GCResults & results)
@@ -516,7 +482,7 @@ bool LocalStore::isActiveTempFile(const GCState & state,
 
 void LocalStore::deleteGarbage(GCState & state, const Path & path)
 {
-    unsigned long long bytesFreed;
+    uint64_t bytesFreed;
     deletePath(path, bytesFreed);
     state.results.bytesFreed += bytesFreed;
 }
@@ -526,7 +492,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
 {
     checkInterrupt();
 
-    unsigned long long size = 0;
+    uint64_t size = 0;
 
     auto storePath = maybeParseStorePath(path);
     if (storePath && isValidPath(*storePath)) {
@@ -608,9 +574,12 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
     /* If keep-derivations is set and this is a derivation, then
        don't delete the derivation if any of the outputs are alive. */
     if (state.gcKeepDerivations && path.isDerivation()) {
-        for (auto & i : queryDerivationOutputs(path))
-            if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                incoming.insert(i);
+        for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
+            if (maybeOutPath &&
+                isValidPath(*maybeOutPath) &&
+                queryPathInfo(*maybeOutPath)->deriver == path
+                )
+                incoming.insert(*maybeOutPath);
     }
 
     /* If keep-outputs is set, then don't delete this path if there
@@ -685,7 +654,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
     AutoCloseDir dir(opendir(linksDir.c_str()));
     if (!dir) throw SysError("opening directory '%1%'", linksDir);
 
-    long long actualSize = 0, unsharedSize = 0;
+    int64_t actualSize = 0, unsharedSize = 0;
 
     struct dirent * dirent;
     while (errno = 0, dirent = readdir(dir.get())) {
@@ -694,9 +663,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
         if (name == "." || name == "..") continue;
         Path path = linksDir + "/" + name;
 
-        struct stat st;
-        if (lstat(path.c_str(), &st) == -1)
-            throw SysError("statting '%1%'", path);
+        auto st = lstat(path);
 
         if (st.st_nlink != 1) {
             actualSize += st.st_size;
@@ -715,10 +682,10 @@ void LocalStore::removeUnusedLinks(const GCState & state)
     struct stat st;
     if (stat(linksDir.c_str(), &st) == -1)
         throw SysError("statting '%1%'", linksDir);
-    long long overhead = st.st_blocks * 512ULL;
+    auto overhead = st.st_blocks * 512ULL;
 
-    printInfo(format("note: currently hard linking saves %.2f MiB")
-        % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
+    printInfo("note: currently hard linking saves %.2f MiB",
+        ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
 }
 
 

src/libstore/globals.cc

@@ -2,6 +2,7 @@
 #include "util.hh"
 #include "archive.hh"
 #include "args.hh"
+#include "abstract-setting-to-json.hh"
 
 #include <algorithm>
 #include <map>
@@ -9,6 +10,8 @@
 #include <dlfcn.h>
 #include <sys/utsname.h>
 
+#include <nlohmann/json.hpp>
+
 
 namespace nix {
 
@@ -22,7 +25,7 @@ namespace nix {
 
 Settings settings;
 
-static GlobalConfig::Register r1(&settings);
+static GlobalConfig::Register rSettings(&settings);
 
 Settings::Settings()
     : nixPrefix(NIX_PREFIX)
@@ -39,6 +42,7 @@ Settings::Settings()
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
     lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
+    allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
 
     caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
     if (caFile == "") {
@@ -144,6 +148,12 @@ bool Settings::isWSL1()
 
 const string nixVersion = PACKAGE_VERSION;
 
+NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
+    {SandboxMode::smEnabled, true},
+    {SandboxMode::smRelaxed, "relaxed"},
+    {SandboxMode::smDisabled, false},
+});
+
 template<> void BaseSetting<SandboxMode>::set(const std::string & str)
 {
     if (str == "true") value = smEnabled;
@@ -160,11 +170,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const
     else abort();
 }
 
-template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
-{
-    AbstractSetting::toJSON(out);
-}
-
 template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
 {
     args.addFlag({

src/libstore/globals.hh

@@ -80,73 +80,178 @@ public:
     Setting<bool> keepGoing{this, false, "keep-going",
         "Whether to keep building derivations when another build fails."};
 
-    Setting<bool> tryFallback{this, false, "fallback",
-        "Whether to fall back to building when substitution fails.",
+    Setting<bool> tryFallback{
+        this, false, "fallback",
+        R"(
+          If set to `true`, Nix will fall back to building from source if a
+          binary substitute fails. This is equivalent to the `--fallback`
+          flag. The default is `false`.
+        )",
         {"build-fallback"}};
 
     /* Whether to show build log output in real time. */
     bool verboseBuild = true;
 
     Setting<size_t> logLines{this, 10, "log-lines",
-        "If verbose-build is false, the number of lines of the tail of "
+        "If `verbose-build` is false, the number of lines of the tail of "
         "the log to show if a build fails."};
 
-    MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs",
-        "Maximum number of parallel build jobs. \"auto\" means use number of cores.",
+    MaxBuildJobsSetting maxBuildJobs{
+        this, 1, "max-jobs",
+        R"(
+          This option defines the maximum number of jobs that Nix will try to
+          build in parallel. The default is `1`. The special value `auto`
+          causes Nix to use the number of CPUs in your system. `0` is useful
+          when using remote builders to prevent any local builds (except for
+          `preferLocalBuild` derivation attribute which executes locally
+          regardless). It can be overridden using the `--max-jobs` (`-j`)
+          command line switch.
+        )",
         {"build-max-jobs"}};
 
-    Setting<unsigned int> buildCores{this, getDefaultCores(), "cores",
-        "Number of CPU cores to utilize in parallel within a build, "
-        "i.e. by passing this number to Make via '-j'. 0 means that the "
-        "number of actual CPU cores on the local host ought to be "
-        "auto-detected.", {"build-cores"}};
+    Setting<unsigned int> buildCores{
+        this, getDefaultCores(), "cores",
+        R"(
+          Sets the value of the `NIX_BUILD_CORES` environment variable in the
+          invocation of builders. Builders can use this variable at their
+          discretion to control the maximum amount of parallelism. For
+          instance, in Nixpkgs, if the derivation attribute
+          `enableParallelBuilding` is set to `true`, the builder passes the
+          `-jN` flag to GNU Make. It can be overridden using the `--cores`
+          command line switch and defaults to `1`. The value `0` means that
+          the builder should use all available CPU cores in the system.
+        )",
+        {"build-cores"}};
 
     /* Read-only mode.  Don't copy stuff to the store, don't change
        the database. */
     bool readOnlyMode = false;
 
-    Setting<std::string> thisSystem{this, SYSTEM, "system",
-        "The canonical Nix system name."};
+    Setting<std::string> thisSystem{
+        this, SYSTEM, "system",
+        R"(
+          This option specifies the canonical Nix system name of the current
+          installation, such as `i686-linux` or `x86_64-darwin`. Nix can only
+          build derivations whose `system` attribute equals the value
+          specified here. In general, it never makes sense to modify this
+          value from its default, since you can use it to ‘lie’ about the
+          platform you are building on (e.g., perform a Mac OS build on a
+          Linux machine; the result would obviously be wrong). It only makes
+          sense if the Nix binaries can run on multiple platforms, e.g.,
+          ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`.
 
-    Setting<time_t> maxSilentTime{this, 0, "max-silent-time",
-        "The maximum time in seconds that a builer can go without "
-        "producing any output on stdout/stderr before it is killed. "
-        "0 means infinity.",
+          It defaults to the canonical Nix system name detected by `configure`
+          at build time.
+        )"};
+
+    Setting<time_t> maxSilentTime{
+        this, 0, "max-silent-time",
+        R"(
+          This option defines the maximum number of seconds that a builder can
+          go without producing any data on standard output or standard error.
+          This is useful (for instance in an automated build system) to catch
+          builds that are stuck in an infinite loop, or to catch remote builds
+          that are hanging due to network problems. It can be overridden using
+          the `--max-silent-time` command line switch.
+
+          The value `0` means that there is no timeout. This is also the
+          default.
+        )",
         {"build-max-silent-time"}};
 
-    Setting<time_t> buildTimeout{this, 0, "timeout",
-        "The maximum duration in seconds that a builder can run. "
-        "0 means infinity.", {"build-timeout"}};
+    Setting<time_t> buildTimeout{
+        this, 0, "timeout",
+        R"(
+          This option defines the maximum number of seconds that a builder can
+          run. This is useful (for instance in an automated build system) to
+          catch builds that are stuck in an infinite loop but keep writing to
+          their standard output or standard error. It can be overridden using
+          the `--timeout` command line switch.
+
+          The value `0` means that there is no timeout. This is also the
+          default.
+        )",
+        {"build-timeout"}};
 
     PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
         "The path of the helper program that executes builds to remote machines."};
 
-    Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
-        "A semicolon-separated list of build machines, in the format of nix.machines."};
+    Setting<std::string> builders{
+        this, "@" + nixConfDir + "/machines", "builders",
+        "A semicolon-separated list of build machines, in the format of `nix.machines`."};
 
-    Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes",
-        "Whether build machines should use their own substitutes for obtaining "
-        "build dependencies if possible, rather than waiting for this host to "
-        "upload them."};
+    Setting<bool> buildersUseSubstitutes{
+        this, false, "builders-use-substitutes",
+        R"(
+          If set to `true`, Nix will instruct remote build machines to use
+          their own binary substitutes if available. In practical terms, this
+          means that remote hosts will fetch as many build dependencies as
+          possible from their own substitutes (e.g, from `cache.nixos.org`),
+          instead of waiting for this host to upload them all. This can
+          drastically reduce build times if the network connection between
+          this computer and the remote build host is slow.
+        )"};
 
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
-    Setting<bool> fsyncMetadata{this, true, "fsync-metadata",
-        "Whether SQLite should use fsync()."};
+    Setting<bool> fsyncMetadata{
+        this, true, "fsync-metadata",
+        R"(
+          If set to `true`, changes to the Nix store metadata (in
+          `/nix/var/nix/db`) are synchronously flushed to disk. This improves
+          robustness in case of system crashes, but reduces performance. The
+          default is `true`.
+        )"};
 
     Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal",
         "Whether SQLite should use WAL mode."};
 
     Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
-        "Whether to call sync() before registering a path as valid."};
+        "Whether to call `sync()` before registering a path as valid."};
 
-    Setting<bool> useSubstitutes{this, true, "substitute",
-        "Whether to use substitutes.",
+    Setting<bool> useSubstitutes{
+        this, true, "substitute",
+        R"(
+          If set to `true` (default), Nix will use binary substitutes if
+          available. This option can be disabled to force building from
+          source.
+        )",
         {"build-use-substitutes"}};
 
-    Setting<std::string> buildUsersGroup{this, "", "build-users-group",
-        "The Unix group that contains the build users."};
+    Setting<std::string> buildUsersGroup{
+        this, "", "build-users-group",
+        R"(
+          This options specifies the Unix group containing the Nix build user
+          accounts. In multi-user Nix installations, builds should not be
+          performed by the Nix account since that would allow users to
+          arbitrarily modify the Nix store and database by supplying specially
+          crafted builders; and they cannot be performed by the calling user
+          since that would allow him/her to influence the build result.
+
+          Therefore, if this option is non-empty and specifies a valid group,
+          builds will be performed under the user accounts that are a member
+          of the group specified here (as listed in `/etc/group`). Those user
+          accounts should not be used for any other purpose\!
+
+          Nix will never run two builds under the same user account at the
+          same time. This is to prevent an obvious security hole: a malicious
+          user writing a Nix expression that modifies the build result of a
+          legitimate Nix expression being built by another user. Therefore it
+          is good to have as many Nix build user accounts as you can spare.
+          (Remember: uids are cheap.)
+
+          The build users should have permission to create files in the Nix
+          store, but not delete them. Therefore, `/nix/store` should be owned
+          by the Nix account, its group should be the group specified here,
+          and its mode should be `1775`.
+
+          If the build users group is empty, builds will be performed under
+          the uid of the Nix process (that is, the uid of the caller if
+          `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
+          `NIX_REMOTE` is `daemon`). Obviously, this should not be used in
+          multi-user settings with untrusted users.
+        )"};
 
     #if __linux__
     Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids",
@@ -165,17 +270,32 @@ public:
         "Whether to impersonate a Linux 2.6 machine on newer kernels.",
         {"build-impersonate-linux-26"}};
 
-    Setting<bool> keepLog{this, true, "keep-build-log",
-        "Whether to store build logs.",
+    Setting<bool> keepLog{
+        this, true, "keep-build-log",
+        R"(
+          If set to `true` (the default), Nix will write the build log of a
+          derivation (i.e. the standard output and error of its builder) to
+          the directory `/nix/var/log/nix/drvs`. The build log can be
+          retrieved using the command `nix-store -l path`.
+        )",
         {"build-keep-log"}};
 
-    Setting<bool> compressLog{this, true, "compress-build-log",
-        "Whether to compress logs.",
+    Setting<bool> compressLog{
+        this, true, "compress-build-log",
+        R"(
+          If set to `true` (the default), build logs written to
+          `/nix/var/log/nix/drvs` will be compressed on the fly using bzip2.
+          Otherwise, they will not be compressed.
+        )",
         {"build-compress-log"}};
 
-    Setting<unsigned long> maxLogSize{this, 0, "max-build-log-size",
-        "Maximum number of bytes a builder can write to stdout/stderr "
-        "before being killed (0 means no limit).",
+    Setting<unsigned long> maxLogSize{
+        this, 0, "max-build-log-size",
+        R"(
+          This option defines the maximum number of bytes that a builder can
+          write to its stdout/stderr. If the builder exceeds this limit, it’s
+          killed. A value of `0` (the default) means that there is no limit.
+        )",
         {"build-max-log-size"}};
 
     /* When buildRepeat > 0 and verboseBuild == true, whether to print
@@ -186,57 +306,156 @@ public:
     Setting<unsigned int> pollInterval{this, 5, "build-poll-interval",
         "How often (in seconds) to poll for locks."};
 
-    Setting<bool> checkRootReachability{this, false, "gc-check-reachability",
-        "Whether to check if new GC roots can in fact be found by the "
-        "garbage collector."};
+    Setting<bool> gcKeepOutputs{
+        this, false, "keep-outputs",
+        R"(
+          If `true`, the garbage collector will keep the outputs of
+          non-garbage derivations. If `false` (default), outputs will be
+          deleted unless they are GC roots themselves (or reachable from other
+          roots).
 
-    Setting<bool> gcKeepOutputs{this, false, "keep-outputs",
-        "Whether the garbage collector should keep outputs of live derivations.",
+          In general, outputs must be registered as roots separately. However,
+          even if the output of a derivation is registered as a root, the
+          collector will still delete store paths that are used only at build
+          time (e.g., the C compiler, or source tarballs downloaded from the
+          network). To prevent it from doing so, set this option to `true`.
+        )",
         {"gc-keep-outputs"}};
 
-    Setting<bool> gcKeepDerivations{this, true, "keep-derivations",
-        "Whether the garbage collector should keep derivers of live paths.",
+    Setting<bool> gcKeepDerivations{
+        this, true, "keep-derivations",
+        R"(
+          If `true` (default), the garbage collector will keep the derivations
+          from which non-garbage store paths were built. If `false`, they will
+          be deleted unless explicitly registered as a root (or reachable from
+          other roots).
+
+          Keeping derivation around is useful for querying and traceability
+          (e.g., it allows you to ask with what dependencies or options a
+          store path was built), so by default this option is on. Turn it off
+          to save a bit of disk space (or a lot if `keep-outputs` is also
+          turned on).
+        )",
         {"gc-keep-derivations"}};
 
-    Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store",
-        "Whether to automatically replace files with identical contents with hard links."};
+    Setting<bool> autoOptimiseStore{
+        this, false, "auto-optimise-store",
+        R"(
+          If set to `true`, Nix automatically detects files in the store
+          that have identical contents, and replaces them with hard links to
+          a single copy. This saves disk space. If set to `false` (the
+          default), you can still run `nix-store --optimise` to get rid of
+          duplicate files.
+        )"};
 
-    Setting<bool> envKeepDerivations{this, false, "keep-env-derivations",
-        "Whether to add derivations as a dependency of user environments "
-        "(to prevent them from being GCed).",
+    Setting<bool> envKeepDerivations{
+        this, false, "keep-env-derivations",
+        R"(
+          If `false` (default), derivations are not stored in Nix user
+          environments. That is, the derivations of any build-time-only
+          dependencies may be garbage-collected.
+
+          If `true`, when you add a Nix derivation to a user environment, the
+          path of the derivation is stored in the user environment. Thus, the
+          derivation will not be garbage-collected until the user environment
+          generation is deleted (`nix-env --delete-generations`). To prevent
+          build-time-only dependencies from being collected, you should also
+          turn on `keep-outputs`.
+
+          The difference between this option and `keep-derivations` is that
+          this one is “sticky”: it applies to any user environment created
+          while this option was enabled, while `keep-derivations` only applies
+          at the moment the garbage collector is run.
+        )",
         {"env-keep-derivations"}};
 
     /* Whether to lock the Nix client and worker to the same CPU. */
     bool lockCPU;
 
-    Setting<SandboxMode> sandboxMode{this,
+    Setting<SandboxMode> sandboxMode{
+        this,
         #if __linux__
           smEnabled
         #else
           smDisabled
         #endif
         , "sandbox",
-        "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".",
+        R"(
+          If set to `true`, builds will be performed in a *sandboxed
+          environment*, i.e., they’re isolated from the normal file system
+          hierarchy and will only see their dependencies in the Nix store,
+          the temporary build directory, private versions of `/proc`,
+          `/dev`, `/dev/shm` and `/dev/pts` (on Linux), and the paths
+          configured with the `sandbox-paths` option. This is useful to
+          prevent undeclared dependencies on files in directories such as
+          `/usr/bin`. In addition, on Linux, builds run in private PID,
+          mount, network, IPC and UTS namespaces to isolate them from other
+          processes in the system (except that fixed-output derivations do
+          not run in private network namespace to ensure they can access the
+          network).
+
+          Currently, sandboxing only work on Linux and macOS. The use of a
+          sandbox requires that Nix is run as root (so you should use the
+          “build users” feature to perform the actual builds under different
+          users than root).
+
+          If this option is set to `relaxed`, then fixed-output derivations
+          and derivations that have the `__noChroot` attribute set to `true`
+          do not run in sandboxes.
+
+          The default is `true` on Linux and `false` on all other platforms.
+        )",
         {"build-use-chroot", "build-use-sandbox"}};
 
-    Setting<PathSet> sandboxPaths{this, {}, "sandbox-paths",
-        "The paths to make available inside the build sandbox.",
+    Setting<PathSet> sandboxPaths{
+        this, {}, "sandbox-paths",
+        R"(
+          A list of paths bind-mounted into Nix sandbox environments. You can
+          use the syntax `target=source` to mount a path in a different
+          location in the sandbox; for instance, `/bin=/nix-bin` will mount
+          the path `/nix-bin` as `/bin` inside the sandbox. If *source* is
+          followed by `?`, then it is not an error if *source* does not exist;
+          for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will
+          only be mounted in the sandbox if it exists in the host filesystem.
+
+          Depending on how Nix was built, the default value for this option
+          may be empty or provide `/bin/sh` as a bind-mount of `bash`.
+        )",
         {"build-chroot-dirs", "build-sandbox-paths"}};
 
     Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
         "Whether to disable sandboxing when the kernel doesn't allow it."};
 
-    Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths",
-        "Additional paths to make available inside the build sandbox.",
+    Setting<PathSet> extraSandboxPaths{
+        this, {}, "extra-sandbox-paths",
+        R"(
+          A list of additional paths appended to `sandbox-paths`. Useful if
+          you want to extend its default value.
+        )",
         {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
 
-    Setting<size_t> buildRepeat{this, 0, "repeat",
-        "The number of times to repeat a build in order to verify determinism.",
+    Setting<size_t> buildRepeat{
+        this, 0, "repeat",
+        R"(
+          How many times to repeat builds to check whether they are
+          deterministic. The default value is 0. If the value is non-zero,
+          every build is repeated the specified number of times. If the
+          contents of any of the runs differs from the previous ones and
+          `enforce-determinism` is true, the build is rejected and the
+          resulting store paths are not registered as “valid” in Nix’s
+          database.
+        )",
         {"build-repeat"}};
 
 #if __linux__
-    Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size",
-        "The size of /dev/shm in the build sandbox."};
+    Setting<std::string> sandboxShmSize{
+        this, "50%", "sandbox-dev-shm-size",
+        R"(
+          This option determines the maximum size of the `tmpfs` filesystem
+          mounted on `/dev/shm` in Linux sandboxes. For the format, see the
+          description of the `size` option of `tmpfs` in mount8. The default
+          is `50%`.
+        )"};
 
     Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
         "The build directory inside the sandbox."};
@@ -250,121 +469,457 @@ public:
         "Whether to log Darwin sandbox access violations to the system log."};
 #endif
 
-    Setting<bool> runDiffHook{this, false, "run-diff-hook",
-        "Whether to run the program specified by the diff-hook setting "
-        "repeated builds produce a different result. Typically used to "
-        "plug in diffoscope."};
+    Setting<bool> runDiffHook{
+        this, false, "run-diff-hook",
+        R"(
+          If true, enable the execution of the `diff-hook` program.
 
-    PathSetting diffHook{this, true, "", "diff-hook",
-        "A program that prints out the differences between the two paths "
-        "specified on its command line."};
+          When using the Nix daemon, `run-diff-hook` must be set in the
+          `nix.conf` configuration file, and cannot be passed at the command
+          line.
+        )"};
 
-    Setting<bool> enforceDeterminism{this, true, "enforce-determinism",
-        "Whether to fail if repeated builds produce different output."};
+    PathSetting diffHook{
+        this, true, "", "diff-hook",
+        R"(
+          Absolute path to an executable capable of diffing build
+          results. The hook is executed if `run-diff-hook` is true, and the
+          output of a build is known to not be the same. This program is not
+          executed to determine if two results are the same.
 
-    Setting<Strings> trustedPublicKeys{this,
+          The diff hook is executed by the same user and group who ran the
+          build. However, the diff hook does not have write access to the
+          store path just built.
+
+          The diff hook program receives three parameters:
+
+          1.  A path to the previous build's results
+
+          2.  A path to the current build's results
+
+          3.  The path to the build's derivation
+
+          4.  The path to the build's scratch directory. This directory will
+              exist only if the build was run with `--keep-failed`.
+
+          The stderr and stdout output from the diff hook will not be
+          displayed to the user. Instead, it will print to the nix-daemon's
+          log.
+
+          When using the Nix daemon, `diff-hook` must be set in the `nix.conf`
+          configuration file, and cannot be passed at the command line.
+        )"};
+
+    Setting<bool> enforceDeterminism{
+        this, true, "enforce-determinism",
+        "Whether to fail if repeated builds produce different output. See `repeat`."};
+
+    Setting<Strings> trustedPublicKeys{
+        this,
         {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
         "trusted-public-keys",
-        "Trusted public keys for secure substitution.",
+        R"(
+          A whitespace-separated list of public keys. When paths are copied
+          from another Nix store (such as a binary cache), they must be
+          signed with one of these keys. For example:
+          `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+          hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`.
+        )",
         {"binary-cache-public-keys"}};
 
-    Setting<Strings> secretKeyFiles{this, {}, "secret-key-files",
-        "Secret keys with which to sign local builds."};
+    Setting<Strings> secretKeyFiles{
+        this, {}, "secret-key-files",
+        R"(
+          A whitespace-separated list of files containing secret (private)
+          keys. These are used to sign locally-built paths. They can be
+          generated using `nix-store --generate-binary-cache-key`. The
+          corresponding public key can be distributed to other users, who
+          can add it to `trusted-public-keys` in their `nix.conf`.
+        )"};
 
-    Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
-        "How long downloaded files are considered up-to-date."};
+    Setting<unsigned int> tarballTtl{
+        this, 60 * 60, "tarball-ttl",
+        R"(
+          The number of seconds a downloaded tarball is considered fresh. If
+          the cached tarball is stale, Nix will check whether it is still up
+          to date using the ETag header. Nix will download a new version if
+          the ETag header is unsupported, or the cached ETag doesn't match.
 
-    Setting<bool> requireSigs{this, true, "require-sigs",
-        "Whether to check that any non-content-addressed path added to the "
-        "Nix store has a valid signature (that is, one signed using a key "
-        "listed in 'trusted-public-keys'."};
+          Setting the TTL to `0` forces Nix to always check if the tarball is
+          up to date.
 
-    Setting<StringSet> extraPlatforms{this,
+          Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`.
+
+          Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`,
+          `fetchTarball`, and `fetchurl` respect this TTL.
+        )"};
+
+    Setting<bool> requireSigs{
+        this, true, "require-sigs",
+        R"(
+          If set to `true` (the default), any non-content-addressed path added
+          or copied to the Nix store (e.g. when substituting from a binary
+          cache) must have a valid signature, that is, be signed using one of
+          the keys listed in `trusted-public-keys` or `secret-key-files`. Set
+          to `false` to disable signature checking.
+        )"};
+
+    Setting<StringSet> extraPlatforms{
+        this,
         std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
         "extra-platforms",
-        "Additional platforms that can be built on the local system. "
-        "These may be supported natively (e.g. armv7 on some aarch64 CPUs "
-        "or using hacks like qemu-user."};
+        R"(
+          Platforms other than the native one which this machine is capable of
+          building for. This can be useful for supporting additional
+          architectures on compatible machines: i686-linux can be built on
+          x86\_64-linux machines (and the default for this setting reflects
+          this); armv7 is backwards-compatible with armv6 and armv5tel; some
+          aarch64 machines can also natively run 32-bit ARM code; and
+          qemu-user may be used to support non-native platforms (though this
+          may be slow and buggy). Most values for this are not enabled by
+          default because build systems will often misdetect the target
+          platform and generate incompatible code, so you may wish to
+          cross-check the results of using this option against proper
+          natively-built versions of your derivations.
+        )"};
 
-    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
+    Setting<StringSet> systemFeatures{
+        this, getDefaultSystemFeatures(),
         "system-features",
-        "Optional features that this system implements (like \"kvm\")."};
+        R"(
+          A set of system “features” supported by this machine, e.g. `kvm`.
+          Derivations can express a dependency on such features through the
+          derivation attribute `requiredSystemFeatures`. For example, the
+          attribute
 
-    Setting<Strings> substituters{this,
+              requiredSystemFeatures = [ "kvm" ];
+
+          ensures that the derivation can only be built on a machine with the
+          `kvm` feature.
+
+          This setting by default includes `kvm` if `/dev/kvm` is accessible,
+          and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
+          that are used in Nixpkgs to route builds to specific machines.
+        )"};
+
+    Setting<Strings> substituters{
+        this,
         nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
         "substituters",
-        "The URIs of substituters (such as https://cache.nixos.org/).",
+        R"(
+          A list of URLs of substituters, separated by whitespace. The default
+          is `https://cache.nixos.org`.
+        )",
         {"binary-caches"}};
 
     // FIXME: provide a way to add to option values.
-    Setting<Strings> extraSubstituters{this, {}, "extra-substituters",
-        "Additional URIs of substituters.",
+    Setting<Strings> extraSubstituters{
+        this, {}, "extra-substituters",
+        R"(
+          Additional binary caches appended to those specified in
+          `substituters`. When used by unprivileged users, untrusted
+          substituters (i.e. those not listed in `trusted-substituters`) are
+          silently ignored.
+        )",
         {"extra-binary-caches"}};
 
-    Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters",
-        "Disabled substituters that may be enabled via the substituters option by untrusted users.",
+    Setting<StringSet> trustedSubstituters{
+        this, {}, "trusted-substituters",
+        R"(
+          A list of URLs of substituters, separated by whitespace. These are
+          not used by default, but can be enabled by users of the Nix daemon
+          by specifying `--option substituters urls` on the command
+          line. Unprivileged users are only allowed to pass a subset of the
+          URLs listed in `substituters` and `trusted-substituters`.
+        )",
         {"trusted-binary-caches"}};
 
-    Setting<Strings> trustedUsers{this, {"root"}, "trusted-users",
-        "Which users or groups are trusted to ask the daemon to do unsafe things."};
+    Setting<Strings> trustedUsers{
+        this, {"root"}, "trusted-users",
+        R"(
+          A list of names of users (separated by whitespace) that have
+          additional rights when connecting to the Nix daemon, such as the
+          ability to specify additional binary caches, or to import unsigned
+          NARs. You can also specify groups by prefixing them with `@`; for
+          instance, `@wheel` means all users in the `wheel` group. The default
+          is `root`.
 
-    Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl",
-        "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that "
-        "return an invalid path result"};
+          > **Warning**
+          > 
+          > Adding a user to `trusted-users` is essentially equivalent to
+          > giving that user root access to the system. For example, the user
+          > can set `sandbox-paths` and thereby obtain read access to
+          > directories that are otherwise inacessible to them.
+        )"};
 
-    Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
-        "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that "
-        "return a valid path result."};
+    Setting<unsigned int> ttlNegativeNarInfoCache{
+        this, 3600, "narinfo-cache-negative-ttl",
+        R"(
+          The TTL in seconds for negative lookups. If a store path is queried
+          from a substituter but was not found, there will be a negative
+          lookup cached in the local disk cache database for the specified
+          duration.
+        )"};
+
+    Setting<unsigned int> ttlPositiveNarInfoCache{
+        this, 30 * 24 * 3600, "narinfo-cache-positive-ttl",
+        R"(
+          The TTL in seconds for positive lookups. If a store path is queried
+          from a substituter, the result of the query will be cached in the
+          local disk cache database including some of the NAR metadata. The
+          default TTL is a month, setting a shorter TTL for positive lookups
+          can be useful for binary caches that have frequent garbage
+          collection, in which case having a more frequent cache invalidation
+          would prevent trying to pull the path again and failing with a hash
+          mismatch if the build isn't reproducible.
+        )"};
 
     /* ?Who we trust to use the daemon in safe ways */
-    Setting<Strings> allowedUsers{this, {"*"}, "allowed-users",
-        "Which users or groups are allowed to connect to the daemon."};
+    Setting<Strings> allowedUsers{
+        this, {"*"}, "allowed-users",
+        R"(
+          A list of names of users (separated by whitespace) that are allowed
+          to connect to the Nix daemon. As with the `trusted-users` option,
+          you can specify groups by prefixing them with `@`. Also, you can
+          allow all users by specifying `*`. The default is `*`.
+
+          Note that trusted users are always allowed to connect.
+        )"};
 
     Setting<bool> printMissing{this, true, "print-missing",
         "Whether to print what paths need to be built or downloaded."};
 
-    Setting<std::string> preBuildHook{this, "",
-        "pre-build-hook",
-        "A program to run just before a build to set derivation-specific build settings."};
+    Setting<std::string> preBuildHook{
+        this, "", "pre-build-hook",
+        R"(
+          If set, the path to a program that can set extra derivation-specific
+          settings for this system. This is used for settings that can't be
+          captured by the derivation model itself and are too variable between
+          different versions of the same system to be hard-coded into nix.
 
-    Setting<std::string> postBuildHook{this, "", "post-build-hook",
-        "A program to run just after each successful build."};
+          The hook is passed the derivation path and, if sandboxes are
+          enabled, the sandbox directory. It can then modify the sandbox and
+          send a series of commands to modify various settings to stdout. The
+          currently recognized commands are:
 
-    Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
-        "Path to the netrc file used to obtain usernames/passwords for downloads."};
+            - `extra-sandbox-paths`  
+              Pass a list of files and directories to be included in the
+              sandbox for this build. One entry per line, terminated by an
+              empty line. Entries have the same format as `sandbox-paths`.
+        )"};
+
+    Setting<std::string> postBuildHook{
+        this, "", "post-build-hook",
+        R"(
+          Optional. The path to a program to execute after each build.
+
+          This option is only settable in the global `nix.conf`, or on the
+          command line by trusted users.
+
+          When using the nix-daemon, the daemon executes the hook as `root`.
+          If the nix-daemon is not involved, the hook runs as the user
+          executing the nix-build.
+
+            - The hook executes after an evaluation-time build.
+
+            - The hook does not execute on substituted paths.
+
+            - The hook's output always goes to the user's terminal.
+
+            - If the hook fails, the build succeeds but no further builds
+              execute.
+
+            - The hook executes synchronously, and blocks other builds from
+              progressing while it runs.
+
+          The program executes with no arguments. The program's environment
+          contains the following environment variables:
+
+            - `DRV_PATH`  
+              The derivation for the built paths.
+
+              Example:
+              `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv`
+
+            - `OUT_PATHS`  
+              Output paths of the built derivation, separated by a space
+              character.
+
+              Example:
+              `/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
+              /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
+              /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
+              /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
+              /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
+        )"};
+
+    Setting<std::string> netrcFile{
+        this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
+        R"(
+          If set to an absolute path to a `netrc` file, Nix will use the HTTP
+          authentication credentials in this file when trying to download from
+          a remote host through HTTP or HTTPS. Defaults to
+          `$NIX_CONF_DIR/netrc`.
+
+          The `netrc` file consists of a list of accounts in the following
+          format:
+
+              machine my-machine
+              login my-username
+              password my-password
+
+          For the exact syntax, see [the `curl`
+          documentation](https://ec.haxx.se/usingcurl-netrc.html).
+
+          > **Note**
+          > 
+          > This must be an absolute path, and `~` is not resolved. For
+          > example, `~/.netrc` won't resolve to your home directory's
+          > `.netrc`.
+        )"};
 
     /* Path to the SSL CA file used */
     Path caFile;
 
 #if __linux__
-    Setting<bool> filterSyscalls{this, true, "filter-syscalls",
-            "Whether to prevent certain dangerous system calls, such as "
-            "creation of setuid/setgid files or adding ACLs or extended "
-            "attributes. Only disable this if you're aware of the "
-            "security implications."};
+    Setting<bool> filterSyscalls{
+        this, true, "filter-syscalls",
+        R"(
+          Whether to prevent certain dangerous system calls, such as
+          creation of setuid/setgid files or adding ACLs or extended
+          attributes. Only disable this if you're aware of the
+          security implications.
+        )"};
 
-    Setting<bool> allowNewPrivileges{this, false, "allow-new-privileges",
-        "Whether builders can acquire new privileges by calling programs with "
-        "setuid/setgid bits or with file capabilities."};
+    Setting<bool> allowNewPrivileges{
+        this, false, "allow-new-privileges",
+        R"(
+          (Linux-specific.) By default, builders on Linux cannot acquire new
+          privileges by calling setuid/setgid programs or programs that have
+          file capabilities. For example, programs such as `sudo` or `ping`
+          will fail. (Note that in sandbox builds, no such programs are
+          available unless you bind-mount them into the sandbox via the
+          `sandbox-paths` option.) You can allow the use of such programs by
+          enabling this option. This is impure and usually undesirable, but
+          may be useful in certain scenarios (e.g. to spin up containers or
+          set up userspace network interfaces in tests).
+        )"};
 #endif
 
-    Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
-        "A list of servers used by builtins.fetchurl to fetch files by hash."};
+    Setting<Strings> hashedMirrors{
+        this, {}, "hashed-mirrors",
+        R"(
+          A list of web servers used by `builtins.fetchurl` to obtain files by
+          hash. The default is `http://tarballs.nixos.org/`. Given a hash type
+          *ht* and a base-16 hash *h*, Nix will try to download the file from
+          *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
+          they have disappeared from their original URI. For example, given
+          the default mirror `http://tarballs.nixos.org/`, when building the
+          derivation
 
-    Setting<uint64_t> minFree{this, 0, "min-free",
-        "Automatically run the garbage collector when free disk space drops below the specified amount."};
+          ```nix
+          builtins.fetchurl {
+            url = "https://example.org/foo-1.2.3.tar.xz";
+            sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+          }
+          ```
 
-    Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
-        "Stop deleting garbage when free disk space is above the specified amount."};
+          Nix will attempt to download this file from
+          `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae`
+          first. If it is not available there, if will try the original URI.
+        )"};
+
+    Setting<uint64_t> minFree{
+        this, 0, "min-free",
+        R"(
+          When free disk space in `/nix/store` drops below `min-free` during a
+          build, Nix performs a garbage-collection until `max-free` bytes are
+          available or there is no more garbage. A value of `0` (the default)
+          disables this feature.
+        )"};
+
+    Setting<uint64_t> maxFree{
+        this, std::numeric_limits<uint64_t>::max(), "max-free",
+        R"(
+          When a garbage collection is triggered by the `min-free` option, it
+          stops as soon as `max-free` bytes are available. The default is
+          infinity (i.e. delete all garbage).
+        )"};
 
     Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
         "Number of seconds between checking free disk space."};
 
-    Setting<Paths> pluginFiles{this, {}, "plugin-files",
-        "Plugins to dynamically load at nix initialization time."};
+    Setting<Paths> pluginFiles{
+        this, {}, "plugin-files",
+        R"(
+          A list of plugin files to be loaded by Nix. Each of these files will
+          be dlopened by Nix, allowing them to affect execution through static
+          initialization. In particular, these plugins may construct static
+          instances of RegisterPrimOp to add new primops or constants to the
+          expression language, RegisterStoreImplementation to add new store
+          implementations, RegisterCommand to add new subcommands to the `nix`
+          command, and RegisterSetting to add new nix config settings. See the
+          constructors for those types for more details.
 
-    Setting<std::string> githubAccessToken{this, "", "github-access-token",
-        "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
+          Since these files are loaded into the same address space as Nix
+          itself, they must be DSOs compatible with the instance of Nix
+          running at the time (i.e. compiled against the same headers, not
+          linked to any incompatible libraries). They should not be linked to
+          any Nix libs directly, as those will be available already at load
+          time.
+
+          If an entry in the list is a directory, all files in the directory
+          are loaded as plugins (non-recursively).
+        )"};
+
+    Setting<StringMap> accessTokens{this, {}, "access-tokens",
+        R"(
+          Access tokens used to access protected GitHub, GitLab, or
+          other locations requiring token-based authentication.
+
+          Access tokens are specified as a string made up of
+          space-separated `host=token` values.  The specific token
+          used is selected by matching the `host` portion against the
+          "host" specification of the input. The actual use of the
+          `token` value is determined by the type of resource being
+          accessed:
+
+          * Github: the token value is the OAUTH-TOKEN string obtained
+            as the Personal Access Token from the Github server (see
+            https://docs.github.com/en/developers/apps/authorizing-oath-apps).
+
+          * Gitlab: the token value is either the OAuth2 token or the
+            Personal Access Token (these are different types tokens
+            for gitlab, see
+            https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
+            The `token` value should be `type:tokenstring` where
+            `type` is either `OAuth2` or `PAT` to indicate which type
+            of token is being specified.
+
+          Example `~/.config/nix/nix.conf`:
+
+          ```
+          access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
+          ```
+
+          Example `~/code/flake.nix`:
+
+          ```nix
+          input.foo = {
+            type = "gitlab";
+            host = "gitlab.mycompany.com";
+            owner = "mycompany";
+            repo = "pro";
+          };
+          ```
+
+          This example specifies three tokens, one each for accessing
+          github.com, gitlab.mycompany.com, and sourceforge.net.
+
+          The `input.foo` uses the "gitlab" fetcher, which might
+          requires specifying the token type along with the token
+          value.
+          )"};
 
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
@@ -378,6 +933,25 @@ public:
 
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
+
+    Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
+        "Maximum size of NARs before spilling them to disk."};
+
+    Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
+        "Path or URI of the global flake registry."};
+
+    Setting<bool> allowSymlinkedStore{
+        this, false, "allow-symlinked-store",
+        R"(
+          If set to `true`, Nix will stop complaining if the store directory
+          (typically /nix/store) contains symlink components.
+
+          This risks making some builds "impure" because builders sometimes
+          "canonicalise" paths by resolving all symlink components. Problems
+          occur if those builds are then deployed to machines where /nix/store
+          resolves to a different location from that of the build machine. You
+          can enable this setting if you are sure you're not going to do that.
+        )"};
 };
 
 

src/libstore/http-binary-cache-store.cc

@@ -2,12 +2,20 @@
 #include "filetransfer.hh"
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
+#include "callback.hh"
 
 namespace nix {
 
 MakeError(UploadToHTTP, Error);
 
-class HttpBinaryCacheStore : public BinaryCacheStore
+struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
+{
+    using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+
+    const std::string name() override { return "Http Binary Cache Store"; }
+};
+
+class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
 {
 private:
 
@@ -24,9 +32,12 @@ private:
 public:
 
     HttpBinaryCacheStore(
-        const Params & params, const Path & _cacheUri)
-        : BinaryCacheStore(params)
-        , cacheUri(_cacheUri)
+        const std::string & scheme,
+        const Path & _cacheUri,
+        const Params & params)
+        : StoreConfig(params)
+        , BinaryCacheStore(params)
+        , cacheUri(scheme + "://" + _cacheUri)
     {
         if (cacheUri.back() == '/')
             cacheUri.pop_back();
@@ -55,6 +66,14 @@ public:
         }
     }
 
+    static std::set<std::string> uriSchemes()
+    {
+        static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
+        auto ret = std::set<std::string>({"http", "https"});
+        if (forceHttp) ret.insert("file");
+        return ret;
+    }
+
 protected:
 
     void maybeDisable()
@@ -85,7 +104,7 @@ protected:
         checkEnabled();
 
         try {
-            FileTransferRequest request(cacheUri + "/" + path);
+            FileTransferRequest request(makeRequest(path));
             request.head = true;
             getFileTransfer()->download(request);
             return true;
@@ -100,11 +119,11 @@ protected:
     }
 
     void upsertFile(const std::string & path,
-        const std::string & data,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) override
     {
-        auto req = FileTransferRequest(cacheUri + "/" + path);
-        req.data = std::make_shared<string>(data); // FIXME: inefficient
+        auto req = makeRequest(path);
+        req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
         req.mimeType = mimeType;
         try {
             getFileTransfer()->upload(req);
@@ -115,8 +134,11 @@ protected:
 
     FileTransferRequest makeRequest(const std::string & path)
     {
-        FileTransferRequest request(cacheUri + "/" + path);
-        return request;
+        return FileTransferRequest(
+            hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://")
+            ? path
+            : cacheUri + "/" + path);
+
     }
 
     void getFile(const std::string & path, Sink & sink) override
@@ -159,18 +181,6 @@ protected:
 
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
-    if (std::string(uri, 0, 7) != "http://" &&
-        std::string(uri, 0, 8) != "https://" &&
-        (!forceHttp || std::string(uri, 0, 7) != "file://"))
-        return 0;
-    auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
-    store->init();
-    return store;
-});
+static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore;
 
 }

src/libstore/legacy-ssh-store.cc

@@ -6,21 +6,28 @@
 #include "worker-protocol.hh"
 #include "ssh.hh"
 #include "derivations.hh"
+#include "callback.hh"
 
 namespace nix {
 
-static std::string uriScheme = "ssh://";
-
-struct LegacySSHStore : public Store
+struct LegacySSHStoreConfig : virtual StoreConfig
 {
-    const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
-    const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
-    const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
-    const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
-    const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
+    using StoreConfig::StoreConfig;
+    const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
+    const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
+    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
+    const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
 
+    const std::string name() override { return "Legacy SSH Store"; }
+};
+
+struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
+{
     // Hack for getting remote build log output.
-    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
+    // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
+    // the documentation
+    const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
 
     struct Connection
     {
@@ -37,8 +44,11 @@ struct LegacySSHStore : public Store
 
     SSHMaster master;
 
-    LegacySSHStore(const string & host, const Params & params)
-        : Store(params)
+    static std::set<std::string> uriSchemes() { return {"ssh"}; }
+
+    LegacySSHStore(const string & scheme, const string & host, const Params & params)
+        : StoreConfig(params)
+        , Store(params)
         , host(host)
         , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
@@ -84,7 +94,7 @@ struct LegacySSHStore : public Store
 
     string getUri() override
     {
-        return uriScheme + host;
+        return *uriSchemes().begin() + "://" + host;
     }
 
     void queryPathInfoUncached(const StorePath & path,
@@ -93,6 +103,9 @@ struct LegacySSHStore : public Store
         try {
             auto conn(connections->get());
 
+            /* No longer support missing NAR hash */
+            assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
+
             debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
 
             conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)};
@@ -100,23 +113,27 @@ struct LegacySSHStore : public Store
 
             auto p = readString(conn->from);
             if (p.empty()) return callback(nullptr);
-            auto info = std::make_shared<ValidPathInfo>(parseStorePath(p));
-            assert(path == info->path);
+            auto path2 = parseStorePath(p);
+            assert(path == path2);
+            /* Hash will be set below. FIXME construct ValidPathInfo at end. */
+            auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy);
 
             PathSet references;
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = readStorePaths<StorePathSet>(*this, conn->from);
+            info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
-            if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
+            {
                 auto s = readString(conn->from);
-                info->narHash = s.empty() ? Hash() : Hash(s);
-                info->ca = parseContentAddressOpt(readString(conn->from));
-                info->sigs = readStrings<StringSet>(conn->from);
+                if (s == "")
+                    throw Error("NAR hash is now mandatory");
+                info->narHash = Hash::parseAnyPrefixed(s);
             }
+            info->ca = parseContentAddressOpt(readString(conn->from));
+            info->sigs = readStrings<StringSet>(conn->from);
 
             auto s = readString(conn->from);
             assert(s == "");
@@ -126,8 +143,7 @@ struct LegacySSHStore : public Store
     }
 
     void addToStore(const ValidPathInfo & info, Source & source,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override
+        RepairFlag repair, CheckSigsFlag checkSigs) override
     {
         debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
 
@@ -140,7 +156,7 @@ struct LegacySSHStore : public Store
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            writeStorePaths(*this, conn->to, info.references);
+            worker_proto::write(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -169,7 +185,7 @@ struct LegacySSHStore : public Store
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            writeStorePaths(*this, conn->to, info.references);
+            worker_proto::write(*this, conn->to, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -203,6 +219,24 @@ struct LegacySSHStore : public Store
         const StorePathSet & references, RepairFlag repair) override
     { unsupported("addTextToStore"); }
 
+private:
+
+    void putBuildSettings(Connection & conn)
+    {
+        conn.to
+            << settings.maxSilentTime
+            << settings.buildTimeout;
+        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2)
+            conn.to
+                << settings.maxLogSize;
+        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3)
+            conn.to
+                << settings.buildRepeat
+                << settings.enforceDeterminism;
+    }
+
+public:
+
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
     {
@@ -212,16 +246,8 @@ struct LegacySSHStore : public Store
             << cmdBuildDerivation
             << printStorePath(drvPath);
         writeDerivation(conn->to, *this, drv);
-        conn->to
-            << settings.maxSilentTime
-            << settings.buildTimeout;
-        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
-            conn->to
-                << settings.maxLogSize;
-        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
-            conn->to
-                << settings.buildRepeat
-                << settings.enforceDeterminism;
+
+        putBuildSettings(*conn);
 
         conn->to.flush();
 
@@ -235,6 +261,29 @@ struct LegacySSHStore : public Store
         return status;
     }
 
+    void buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) override
+    {
+        auto conn(connections->get());
+
+        conn->to << cmdBuildPaths;
+        Strings ss;
+        for (auto & p : drvPaths)
+            ss.push_back(p.to_string(*this));
+        conn->to << ss;
+
+        putBuildSettings(*conn);
+
+        conn->to.flush();
+
+        BuildResult result;
+        result.status = (BuildResult::Status) readInt(conn->from);
+
+        if (!result.success()) {
+            conn->from >> result.errorMsg;
+            throw Error(result.status, result.errorMsg);
+        }
+    }
+
     void ensurePath(const StorePath & path) override
     { unsupported("ensurePath"); }
 
@@ -252,10 +301,10 @@ struct LegacySSHStore : public Store
         conn->to
             << cmdQueryClosure
             << includeOutputs;
-        writeStorePaths(*this, conn->to, paths);
+        worker_proto::write(*this, conn->to, paths);
         conn->to.flush();
 
-        for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
+        for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
             out.insert(i);
     }
 
@@ -268,10 +317,10 @@ struct LegacySSHStore : public Store
             << cmdQueryValidPaths
             << false // lock
             << maybeSubstitute;
-        writeStorePaths(*this, conn->to, paths);
+        worker_proto::write(*this, conn->to, paths);
         conn->to.flush();
 
-        return readStorePaths<StorePathSet>(*this, conn->from);
+        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
 
     void connect() override
@@ -286,12 +335,6 @@ struct LegacySSHStore : public Store
     }
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
-    return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
-});
+static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
 
 }

src/libstore/local-binary-cache-store.cc

@@ -4,7 +4,14 @@
 
 namespace nix {
 
-class LocalBinaryCacheStore : public BinaryCacheStore
+struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
+{
+    using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+
+    const std::string name() override { return "Local Binary Cache Store"; }
+};
+
+class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
 {
 private:
 
@@ -13,8 +20,11 @@ private:
 public:
 
     LocalBinaryCacheStore(
-        const Params & params, const Path & binaryCacheDir)
-        : BinaryCacheStore(params)
+        const std::string scheme,
+        const Path & binaryCacheDir,
+        const Params & params)
+        : StoreConfig(params)
+        , BinaryCacheStore(params)
         , binaryCacheDir(binaryCacheDir)
     {
     }
@@ -26,13 +36,25 @@ public:
         return "file://" + binaryCacheDir;
     }
 
+    static std::set<std::string> uriSchemes();
+
 protected:
 
     bool fileExists(const std::string & path) override;
 
     void upsertFile(const std::string & path,
-        const std::string & data,
-        const std::string & mimeType) override;
+        std::shared_ptr<std::basic_iostream<char>> istream,
+        const std::string & mimeType) override
+    {
+        auto path2 = binaryCacheDir + "/" + path;
+        Path tmp = path2 + ".tmp." + std::to_string(getpid());
+        AutoDelete del(tmp, false);
+        StreamToSourceAdapter source(istream);
+        writeFile(tmp, source);
+        if (rename(tmp.c_str(), path2.c_str()))
+            throw SysError("renaming '%1%' to '%2%'", tmp, path2);
+        del.cancel();
+    }
 
     void getFile(const std::string & path, Sink & sink) override
     {
@@ -52,7 +74,9 @@ protected:
             if (entry.name.size() != 40 ||
                 !hasSuffix(entry.name, ".narinfo"))
                 continue;
-            paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
+            paths.insert(parseStorePath(
+                    storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
+                    + "-" + MissingName));
         }
 
         return paths;
@@ -68,38 +92,19 @@ void LocalBinaryCacheStore::init()
     BinaryCacheStore::init();
 }
 
-static void atomicWrite(const Path & path, const std::string & s)
-{
-    Path tmp = path + ".tmp." + std::to_string(getpid());
-    AutoDelete del(tmp, false);
-    writeFile(tmp, s);
-    if (rename(tmp.c_str(), path.c_str()))
-        throw SysError("renaming '%1%' to '%2%'", tmp, path);
-    del.cancel();
-}
-
 bool LocalBinaryCacheStore::fileExists(const std::string & path)
 {
     return pathExists(binaryCacheDir + "/" + path);
 }
 
-void LocalBinaryCacheStore::upsertFile(const std::string & path,
-    const std::string & data,
-    const std::string & mimeType)
+std::set<std::string> LocalBinaryCacheStore::uriSchemes()
 {
-    atomicWrite(binaryCacheDir + "/" + path, data);
+    if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1")
+        return {};
+    else
+        return {"file"};
 }
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
-        std::string(uri, 0, 7) != "file://")
-        return 0;
-    auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
-    store->init();
-    return store;
-});
+static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regLocalBinaryCacheStore;
 
 }

src/libstore/local-fs-store.cc

@@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
 
     Path toRealPath(const Path & path)
     {
-        Path storePath = store->toStorePath(path);
-        if (!store->isValidPath(store->parseStorePath(storePath)))
-            throw InvalidPath("path '%1%' is not a valid store path", storePath);
+        auto storePath = store->toStorePath(path).first;
+        if (!store->isValidPath(storePath))
+            throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
 

src/libstore/local-store.cc

@@ -6,6 +6,7 @@
 #include "derivations.hh"
 #include "nar-info.hh"
 #include "references.hh"
+#include "callback.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -42,7 +43,8 @@ namespace nix {
 
 
 LocalStore::LocalStore(const Params & params)
-    : Store(params)
+    : StoreConfig(params)
+    , Store(params)
     , LocalFSStore(params)
     , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
         "physical path to the Nix store"}
@@ -108,12 +110,11 @@ LocalStore::LocalStore(const Params & params)
     }
 
     /* Ensure that the store and its parents are not symlinks. */
-    if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") {
+    if (!settings.allowSymlinkedStore) {
         Path path = realStoreDir;
         struct stat st;
         while (path != "/") {
-            if (lstat(path.c_str(), &st))
-                throw SysError("getting status of '%1%'", path);
+            st = lstat(path);
             if (S_ISLNK(st.st_mode))
                 throw Error(
                         "the path '%1%' is a symlink; "
@@ -417,10 +418,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
 
 void canonicaliseTimestampAndPermissions(const Path & path)
 {
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
-    canonicaliseTimestampAndPermissions(path, st);
+    canonicaliseTimestampAndPermissions(path, lstat(path));
 }
 
 
@@ -441,9 +439,7 @@ static void canonicalisePathMetaData_(
     }
 #endif
 
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
+    auto st = lstat(path);
 
     /* Really make sure that the path is of a supported type. */
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
@@ -479,8 +475,7 @@ static void canonicalisePathMetaData_(
        ensure that we don't fail on hard links within the same build
        (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
     if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) {
-        assert(!S_ISDIR(st.st_mode));
-        if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
+        if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
             throw BuildError("invalid ownership on file '%1%'", path);
         mode_t mode = st.st_mode & ~S_IFMT;
         assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
@@ -526,9 +521,7 @@ void canonicalisePathMetaData(
 
     /* On platforms that don't have lchown(), the top-level path can't
        be a symlink, since we can't change its ownership. */
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
+    auto st = lstat(path);
 
     if (st.st_uid != geteuid()) {
         assert(S_ISLNK(st.st_mode));
@@ -551,11 +544,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     std::string drvName(drvPath.name());
     drvName = string(drvName, 0, drvName.size() - drvExtension.size());
 
-    auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName)
+    auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
     {
-        if (actual != expected)
-            throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
-                printStorePath(drvPath), printStorePath(actual), printStorePath(expected));
         auto j = drv.env.find(varName);
         if (j == drv.env.end() || parseStorePath(j->second) != actual)
             throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
@@ -563,24 +553,54 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     };
 
 
-    if (drv.isFixedOutput()) {
-        DerivationOutputs::const_iterator out = drv.outputs.find("out");
-        if (out == drv.outputs.end())
-            throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
+    // Don't need the answer, but do this anyways to assert is proper
+    // combination. The code below is more general and naturally allows
+    // combinations that are currently prohibited.
+    drv.type();
 
-        check(
-            makeFixedOutputPath(
-                out->second.hash->method,
-                out->second.hash->hash,
-                drvName),
-            out->second.path, "out");
+    std::optional<Hash> h;
+    for (auto & i : drv.outputs) {
+        std::visit(overloaded {
+            [&](DerivationOutputInputAddressed doia) {
+                if (!h) {
+                    // somewhat expensive so we do lazily
+                    auto temp = hashDerivationModulo(*this, drv, true);
+                    h = std::get<Hash>(temp);
+                }
+                StorePath recomputed = makeOutputPath(i.first, *h, drvName);
+                if (doia.path != recomputed)
+                    throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
+                        printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
+                envHasRightPath(doia.path, i.first);
+            },
+            [&](DerivationOutputCAFixed dof) {
+                StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
+                envHasRightPath(path, i.first);
+            },
+            [&](DerivationOutputCAFloating _) {
+                /* Nothing to check */
+            },
+        }, i.second.output);
     }
+}
+
+
+void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
+{
+    auto state(_state.lock());
+    return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
+}
+
+void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
+{
+    retrySQLite<void>([&]() {
+        state.stmtAddDerivationOutput.use()
+            (deriver)
+            (outputName)
+            (printStorePath(output))
+            .exec();
+    });
 
-    else {
-        Hash h = hashDerivationModulo(*this, drv, true);
-        for (auto & i : drv.outputs)
-            check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
-    }
 }
 
 
@@ -601,7 +621,7 @@ uint64_t LocalStore::addValidPath(State & state,
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
         (renderContentAddress(info.ca), (bool) info.ca)
         .exec();
-    uint64_t id = sqlite3_last_insert_rowid(state.db);
+    uint64_t id = state.db.getLastInsertedRowId();
 
     /* If this is a derivation, then store the derivation outputs in
        the database.  This is useful for the garbage collector: it can
@@ -617,12 +637,11 @@ uint64_t LocalStore::addValidPath(State & state,
            registration above is undone. */
         if (checkOutputs) checkDerivationOutputs(info.path, drv);
 
-        for (auto & i : drv.outputs) {
-            state.stmtAddDerivationOutput.use()
-                (id)
-                (i.first)
-                (printStorePath(i.second.path))
-                .exec();
+        for (auto & i : drv.outputsAndOptPaths(*this)) {
+            /* Floating CA derivations have indeterminate output paths until
+               they are built, so don't register anything in that case */
+            if (i.second.second)
+                linkDeriverToPath(state, id, i.first, *i.second.second);
         }
     }
 
@@ -640,25 +659,28 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        auto info = std::make_shared<ValidPathInfo>(path);
-
         callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
             auto state(_state.lock());
 
             /* Get the path info. */
-            auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path)));
+            auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path)));
 
             if (!useQueryPathInfo.next())
                 return std::shared_ptr<ValidPathInfo>();
 
-            info->id = useQueryPathInfo.getInt(0);
+            auto id = useQueryPathInfo.getInt(0);
 
+            auto narHash = Hash::dummy;
             try {
-                info->narHash = Hash(useQueryPathInfo.getStr(1));
+                narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
             } catch (BadHash & e) {
-                throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what());
+                throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what());
             }
 
+            auto info = std::make_shared<ValidPathInfo>(path, narHash);
+
+            info->id = id;
+
             info->registrationTime = useQueryPathInfo.getInt(2);
 
             auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
@@ -706,7 +728,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
 {
     auto use(state.stmtQueryPathInfo.use()(printStorePath(path)));
     if (!use.next())
-        throw Error("path '%s' is not valid", printStorePath(path));
+        throw InvalidPath("path '%s' is not valid", printStorePath(path));
     return use.getInt(0);
 }
 
@@ -781,17 +803,61 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 }
 
 
-OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
 {
-    return retrySQLite<OutputPathMap>([&]() {
+    auto path = path_;
+    std::map<std::string, std::optional<StorePath>> outputs;
+    Derivation drv = readDerivation(path);
+    for (auto & [outName, _] : drv.outputs) {
+        outputs.insert_or_assign(outName, std::nullopt);
+    }
+    bool haveCached = false;
+    {
+        auto resolutions = drvPathResolutions.lock();
+        auto resolvedPathOptIter = resolutions->find(path);
+        if (resolvedPathOptIter != resolutions->end()) {
+            auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
+            if (resolvedPathOpt)
+                path = *resolvedPathOpt;
+            haveCached = true;
+        }
+    }
+    /* can't just use else-if instead of `!haveCached` because we need to unlock
+       `drvPathResolutions` before it is locked in `Derivation::resolve`. */
+    if (!haveCached && drv.type() == DerivationType::CAFloating) {
+        /* Try resolve drv and use that path instead. */
+        auto attempt = drv.tryResolve(*this);
+        if (!attempt)
+            /* If we cannot resolve the derivation, we cannot have any path
+               assigned so we return the map of all std::nullopts. */
+            return outputs;
+        /* Just compute store path */
+        auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true);
+        /* Store in memo table. */
+        /* FIXME: memo logic should not be local-store specific, should have
+           wrapper-method instead. */
+        drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
+        path = std::move(pathResolved);
+    }
+    return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
         auto state(_state.lock());
 
-        auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
-            (queryValidPathId(*state, path)));
+        uint64_t drvId;
+        try {
+            drvId = queryValidPathId(*state, path);
+        } catch (InvalidPath &) {
+            /* FIXME? if the derivation doesn't exist, we cannot have a mapping
+               for it. */
+            return outputs;
+        }
+
+        auto useQueryDerivationOutputs {
+            state->stmtQueryDerivationOutputs.use()
+            (drvId)
+        };
 
-        OutputPathMap outputs;
         while (useQueryDerivationOutputs.next())
-            outputs.emplace(
+            outputs.insert_or_assign(
                 useQueryDerivationOutputs.getStr(0),
                 parseStorePath(useQueryDerivationOutputs.getStr(1))
             );
@@ -853,20 +919,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
 }
 
 
-void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
-    SubstitutablePathInfos & infos)
+void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
 {
     if (!settings.useSubstitutes) return;
     for (auto & sub : getDefaultSubstituters()) {
-        if (sub->storeDir != storeDir) continue;
         for (auto & path : paths) {
-            if (infos.count(path)) continue;
-            debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
+            auto subPath(path.first);
+
+            // recompute store path so that we can use a different store root
+            if (path.second) {
+                subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
+                if (sub->storeDir == storeDir)
+                    assert(subPath == path.first);
+                if (subPath != path.first)
+                    debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
+            } else if (sub->storeDir != storeDir) continue;
+
+            debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
             try {
-                auto info = sub->queryPathInfo(path);
+                auto info = sub->queryPathInfo(subPath);
+
+                if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
+                    continue;
+
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
                     std::shared_ptr<const ValidPathInfo>(info));
-                infos.insert_or_assign(path, SubstitutablePathInfo{
+                infos.insert_or_assign(path.first, SubstitutablePathInfo{
                     info->deriver,
                     info->references,
                     narInfo ? narInfo->fileSize : 0,
@@ -969,11 +1047,8 @@ const PublicKeys & LocalStore::getPublicKeys()
 
 
 void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
-    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+    RepairFlag repair, CheckSigsFlag checkSigs)
 {
-    if (!info.narHash)
-        throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
-
     if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys()))
         throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path));
 
@@ -983,7 +1058,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
         PathLocks outputLock;
 
-        Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
+        auto realPath = Store::toRealPath(info.path);
 
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
@@ -1008,11 +1083,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             else
                 hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
 
-            LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
-                size_t n = source.read(data, len);
-                (*hashSink)(data, n);
-                return n;
-            });
+            TeeSource wrapperSource { source, *hashSink };
 
             restorePath(realPath, wrapperSource);
 
@@ -1040,12 +1111,64 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 }
 
 
-StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
+StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
     FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
 {
-    Hash h = hashString(hashAlgo, dump);
+    /* For computing the store path. */
+    auto hashSink = std::make_unique<HashSink>(hashAlgo);
+    TeeSource source { source0, *hashSink };
 
-    auto dstPath = makeFixedOutputPath(method, h, name);
+    /* Read the source path into memory, but only if it's up to
+       narBufferSize bytes. If it's larger, write it to a temporary
+       location in the Nix store. If the subsequently computed
+       destination store path is already valid, we just delete the
+       temporary path. Otherwise, we move it to the destination store
+       path. */
+    bool inMemory = false;
+
+    std::string dump;
+
+    /* Fill out buffer, and decide whether we are working strictly in
+       memory based on whether we break out because the buffer is full
+       or the original source is empty */
+    while (dump.size() < settings.narBufferSize) {
+        auto oldSize = dump.size();
+        constexpr size_t chunkSize = 65536;
+        auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
+        dump.resize(oldSize + want);
+        auto got = 0;
+        try {
+            got = source.read((uint8_t *) dump.data() + oldSize, want);
+        } catch (EndOfFile &) {
+            inMemory = true;
+            break;
+        }
+        dump.resize(oldSize + got);
+    }
+
+    std::unique_ptr<AutoDelete> delTempDir;
+    Path tempPath;
+
+    if (!inMemory) {
+        /* Drain what we pulled so far, and then keep on pulling */
+        StringSource dumpSource { dump };
+        ChainSource bothSource { dumpSource, source };
+
+        auto tempDir = createTempDir(realStoreDir, "add");
+        delTempDir = std::make_unique<AutoDelete>(tempDir);
+        tempPath = tempDir + "/x";
+
+        if (method == FileIngestionMethod::Recursive)
+            restorePath(tempPath, bothSource);
+        else
+            writeFile(tempPath, bothSource);
+
+        dump.clear();
+    }
+
+    auto [hash, size] = hashSink->finish();
+
+    auto dstPath = makeFixedOutputPath(method, hash, name);
 
     addTempRoot(dstPath);
 
@@ -1054,8 +1177,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
         /* The first check above is an optimisation to prevent
            unnecessary lock acquisition. */
 
-        Path realPath = realStoreDir + "/";
-        realPath += dstPath.to_string();
+        auto realPath = Store::toRealPath(dstPath);
 
         PathLocks outputLock({realPath});
 
@@ -1065,31 +1187,35 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
             autoGC();
 
-            if (method == FileIngestionMethod::Recursive) {
-                StringSource source(dump);
-                restorePath(realPath, source);
-            } else
-                writeFile(realPath, dump);
+            if (inMemory) {
+                 StringSource dumpSource { dump };
+                /* Restore from the NAR in memory. */
+                if (method == FileIngestionMethod::Recursive)
+                    restorePath(realPath, dumpSource);
+                else
+                    writeFile(realPath, dumpSource);
+            } else {
+                /* Move the temporary path we restored above. */
+                if (rename(tempPath.c_str(), realPath.c_str()))
+                    throw Error("renaming '%s' to '%s'", tempPath, realPath);
+            }
 
-            canonicalisePathMetaData(realPath, {});
+            /* For computing the nar hash. In recursive SHA-256 mode, this
+               is the same as the store hash, so no need to do it again. */
+            auto narHash = std::pair { hash, size };
+            if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
+                HashSink narSink { htSHA256 };
+                dumpPath(realPath, narSink);
+                narHash = narSink.finish();
+            }
 
-            /* Register the SHA-256 hash of the NAR serialisation of
-               the path in the database.  We may just have computed it
-               above (if called with recursive == true and hashAlgo ==
-               sha256); otherwise, compute it here. */
-            HashResult hash;
-            if (method == FileIngestionMethod::Recursive) {
-                hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
-                hash.second = dump.size();
-            } else
-                hash = hashPath(htSHA256, realPath);
+            canonicalisePathMetaData(realPath, {}); // FIXME: merge into restorePath
 
-            optimisePath(realPath); // FIXME: combine with hashPath()
+            optimisePath(realPath);
 
-            ValidPathInfo info(dstPath);
-            info.narHash = hash.first;
-            info.narSize = hash.second;
-            info.ca = FixedOutputHash { .method = method, .hash = h };
+            ValidPathInfo info { dstPath, narHash.first };
+            info.narSize = narHash.second;
+            info.ca = FixedOutputHash { .method = method, .hash = hash };
             registerValidPath(info);
         }
 
@@ -1100,24 +1226,6 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 }
 
 
-StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
-    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
-{
-    Path srcPath(absPath(_srcPath));
-
-    /* Read the whole path into memory. This is not a very scalable
-       method for very large paths, but `copyPath' is mainly used for
-       small files. */
-    StringSink sink;
-    if (method == FileIngestionMethod::Recursive)
-        dumpPath(srcPath, sink, filter);
-    else
-        sink.s = make_ref<std::string>(readFile(srcPath));
-
-    return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
-}
-
-
 StorePath LocalStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
@@ -1128,8 +1236,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
     if (repair || !isValidPath(dstPath)) {
 
-        Path realPath = realStoreDir + "/";
-        realPath += dstPath.to_string();
+        auto realPath = Store::toRealPath(dstPath);
 
         PathLocks outputLock({realPath});
 
@@ -1149,8 +1256,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
             optimisePath(realPath);
 
-            ValidPathInfo info(dstPath);
-            info.narHash = narHash;
+            ValidPathInfo info { dstPath, narHash };
             info.narSize = sink.s->size();
             info.references = references;
             info.ca = TextHash { .hash = hash };
@@ -1265,9 +1371,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 std::unique_ptr<AbstractHashSink> hashSink;
                 if (!info->ca || !info->references.count(info->path))
-                    hashSink = std::make_unique<HashSink>(*info->narHash.type);
+                    hashSink = std::make_unique<HashSink>(info->narHash.type);
                 else
-                    hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
+                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
 
                 dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();
@@ -1387,7 +1493,7 @@ static void makeMutable(const Path & path)
 {
     checkInterrupt();
 
-    struct stat st = lstat(path);
+    auto st = lstat(path);
 
     if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return;
 

src/libstore/local-store.hh

@@ -23,18 +23,26 @@ namespace nix {
 const int nixSchemaVersion = 10;
 
 
-struct Derivation;
-
-
 struct OptimiseStats
 {
     unsigned long filesLinked = 0;
-    unsigned long long bytesFreed = 0;
-    unsigned long long blocksFreed = 0;
+    uint64_t bytesFreed = 0;
+    uint64_t blocksFreed = 0;
+};
+
+struct LocalStoreConfig : virtual LocalFSStoreConfig
+{
+    using LocalFSStoreConfig::LocalFSStoreConfig;
+
+    Setting<bool> requireSigs{(StoreConfig*) this,
+        settings.requireSigs,
+        "require-sigs", "whether store paths should have a trusted signature on import"};
+
+    const std::string name() override { return "Local Store"; }
 };
 
 
-class LocalStore : public LocalFSStore
+class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
 {
 private:
 
@@ -98,10 +106,6 @@ public:
 
 private:
 
-    Setting<bool> requireSigs{(Store*) this,
-        settings.requireSigs,
-        "require-sigs", "whether store paths should have a trusted signature on import"};
-
     const PublicKeys & getPublicKeys();
 
 public:
@@ -133,29 +137,20 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
-    void querySubstitutablePathInfos(const StorePathSet & paths,
+    void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) override;
 
     void addToStore(const ValidPathInfo & info, Source & source,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override;
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method, HashType hashAlgo,
-        PathFilter & filter, RepairFlag repair) override;
-
-    /* Like addToStore(), but the contents of the path are contained
-       in `dump', which is either a NAR serialisation (if recursive ==
-       true) or simply the contents of a regular file (if recursive ==
-       false). */
-    StorePath addToStoreFromDump(const string & dump, const string & name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+    StorePath addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
 
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
@@ -291,6 +286,11 @@ private:
        specified by the ‘secret-key-files’ option. */
     void signPathInfo(ValidPathInfo & info);
 
+    /* Register the store path 'output' as the output named 'outputName' of
+       derivation 'deriver'. */
+    void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
+    void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
+
     Path getRealStoreDir() override { return realStoreDir; }
 
     void createUser(const std::string & userName, uid_t userId) override;

src/libstore/local.mk

@@ -61,3 +61,6 @@ $(d)/build.cc:
 clean-files += $(d)/schema.sql.gen.hh
 
 $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
+
+$(foreach i, $(wildcard src/libstore/builtins/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))

src/libstore/machines.cc

@@ -1,6 +1,7 @@
 #include "machines.hh"
 #include "util.hh"
 #include "globals.hh"
+#include "store-api.hh"
 
 #include <algorithm>
 
@@ -48,6 +49,29 @@ bool Machine::mandatoryMet(const std::set<string> & features) const {
         });
 }
 
+ref<Store> Machine::openStore() const {
+    Store::Params storeParams;
+    if (hasPrefix(storeUri, "ssh://")) {
+        storeParams["max-connections"] = "1";
+        storeParams["log-fd"] = "4";
+        if (sshKey != "")
+            storeParams["ssh-key"] = sshKey;
+    }
+    {
+        auto & fs = storeParams["system-features"];
+        auto append = [&](auto feats) {
+            for (auto & f : feats) {
+                if (fs.size() > 0) fs += ' ';
+                fs += f;
+            }
+        };
+        append(supportedFeatures);
+        append(mandatoryFeatures);
+    }
+
+    return nix::openStore(storeUri, storeParams);
+}
+
 void parseMachines(const std::string & s, Machines & machines)
 {
     for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {

src/libstore/machines.hh

@@ -4,6 +4,8 @@
 
 namespace nix {
 
+class Store;
+
 struct Machine {
 
     const string storeUri;
@@ -28,6 +30,8 @@ struct Machine {
         decltype(supportedFeatures) supportedFeatures,
         decltype(mandatoryFeatures) mandatoryFeatures,
         decltype(sshPublicHostKey) sshPublicHostKey);
+
+    ref<Store> openStore() const;
 };
 
 typedef std::vector<Machine> Machines;

src/libstore/misc.cc

@@ -4,7 +4,8 @@
 #include "local-store.hh"
 #include "store-api.hh"
 #include "thread-pool.hh"
-
+#include "topo-sort.hh"
+#include "callback.hh"
 
 namespace nix {
 
@@ -108,9 +109,19 @@ void Store::computeFSClosure(const StorePath & startPath,
 }
 
 
+std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
+{
+    auto out = drv.outputs.find("out");
+    if (out != drv.outputs.end()) {
+        if (auto v = std::get_if<DerivationOutputCAFixed>(&out->second.output))
+            return v->hash;
+    }
+    return std::nullopt;
+}
+
 void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
     StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
-    unsigned long long & downloadSize_, unsigned long long & narSize_)
+    uint64_t & downloadSize_, uint64_t & narSize_)
 {
     Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
 
@@ -122,8 +133,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
     {
         std::unordered_set<std::string> done;
         StorePathSet & unknown, & willSubstitute, & willBuild;
-        unsigned long long & downloadSize;
-        unsigned long long & narSize;
+        uint64_t & downloadSize;
+        uint64_t & narSize;
     };
 
     struct DrvState
@@ -157,7 +168,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
         auto outPath = parseStorePath(outPathS);
 
         SubstitutablePathInfos infos;
-        querySubstitutablePathInfos({outPath}, infos);
+        querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
 
         if (infos.empty()) {
             drvState_->lock()->done = true;
@@ -192,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
                 return;
             }
 
+            PathSet invalid;
+            /* true for regular derivations, and CA derivations for which we
+               have a trust mapping for all wanted outputs. */
+            auto knownOutputPaths = true;
+            for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
+                if (!pathOpt) {
+                    knownOutputPaths = false;
+                    break;
+                }
+                if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
+                    invalid.insert(printStorePath(*pathOpt));
+            }
+            if (knownOutputPaths && invalid.empty()) return;
+
             auto drv = make_ref<Derivation>(derivationFromPath(path.path));
             ParsedDerivation parsedDrv(StorePath(path.path), *drv);
 
-            PathSet invalid;
-            for (auto & j : drv->outputs)
-                if (wantOutput(j.first, path.outputs)
-                    && !isValidPath(j.second.path))
-                    invalid.insert(printStorePath(j.second.path));
-            if (invalid.empty()) return;
-
-            if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
+            if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
                     pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
@@ -214,7 +232,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
             if (isValidPath(path.path)) return;
 
             SubstitutablePathInfos infos;
-            querySubstitutablePathInfos({path.path}, infos);
+            querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
 
             if (infos.empty()) {
                 auto state(state_.lock());
@@ -246,41 +264,21 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
 
 StorePaths Store::topoSortPaths(const StorePathSet & paths)
 {
-    StorePaths sorted;
-    StorePathSet visited, parents;
-
-    std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit;
-
-    dfsVisit = [&](const StorePath & path, const StorePath * parent) {
-        if (parents.count(path))
-            throw BuildError("cycle detected in the references of '%s' from '%s'",
-                printStorePath(path), printStorePath(*parent));
-
-        if (!visited.insert(path).second) return;
-        parents.insert(path);
-
-        StorePathSet references;
-        try {
-            references = queryPathInfo(path)->references;
-        } catch (InvalidPath &) {
-        }
-
-        for (auto & i : references)
-            /* Don't traverse into paths that don't exist.  That can
-               happen due to substitutes for non-existent paths. */
-            if (i != path && paths.count(i))
-                dfsVisit(i, &path);
-
-        sorted.push_back(path);
-        parents.erase(path);
-    };
-
-    for (auto & i : paths)
-        dfsVisit(i, nullptr);
-
-    std::reverse(sorted.begin(), sorted.end());
-
-    return sorted;
+    return topoSort(paths,
+        {[&](const StorePath & path) {
+            StorePathSet references;
+            try {
+                references = queryPathInfo(path)->references;
+            } catch (InvalidPath &) {
+            }
+            return references;
+        }},
+        {[&](const StorePath & path, const StorePath & parent) {
+            return BuildError(
+                "cycle detected in the references of '%s' from '%s'",
+                printStorePath(path),
+                printStorePath(parent));
+        }});
 }
 
 

src/libstore/names.cc

@@ -1,10 +1,18 @@
 #include "names.hh"
 #include "util.hh"
 
+#include <regex>
+
 
 namespace nix {
 
 
+struct Regex
+{
+    std::regex regex;
+};
+
+
 DrvName::DrvName()
 {
     name = "";
@@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0)
 }
 
 
+DrvName::~DrvName()
+{ }
+
+
 bool DrvName::matches(DrvName & n)
 {
     if (name != "*") {
-        if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended));
-        if (!std::regex_match(n.name, *regex)) return false;
+        if (!regex) {
+            regex = std::make_unique<Regex>();
+            regex->regex = std::regex(name, std::regex::extended);
+        }
+        if (!std::regex_match(n.name, regex->regex)) return false;
     }
     if (version != "" && version != n.version) return false;
     return true;
@@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs)
 {
     DrvNames result;
     for (auto & i : opArgs)
-        result.push_back(DrvName(i));
+        result.emplace_back(i);
     return result;
 }
 

src/libstore/names.hh

@@ -3,10 +3,11 @@
 #include <memory>
 
 #include "types.hh"
-#include <regex>
 
 namespace nix {
 
+struct Regex;
+
 struct DrvName
 {
     string fullName;
@@ -16,10 +17,12 @@ struct DrvName
 
     DrvName();
     DrvName(std::string_view s);
+    ~DrvName();
+
     bool matches(DrvName & n);
 
 private:
-    std::unique_ptr<std::regex> regex;
+    std::unique_ptr<Regex> regex;
 };
 
 typedef list<DrvName> DrvNames;

src/libstore/nar-accessor.cc

@@ -18,7 +18,7 @@ struct NarMember
 
     /* If this is a regular file, position of the contents of this
        file in the NAR. */
-    size_t start = 0, size = 0;
+    uint64_t start = 0, size = 0;
 
     std::string target;
 
@@ -34,20 +34,23 @@ struct NarAccessor : public FSAccessor
 
     NarMember root;
 
-    struct NarIndexer : ParseSink, StringSource
+    struct NarIndexer : ParseSink, Source
     {
         NarAccessor & acc;
+        Source & source;
 
         std::stack<NarMember *> parents;
 
-        std::string currentStart;
         bool isExec = false;
 
-        NarIndexer(NarAccessor & acc, const std::string & nar)
-            : StringSource(nar), acc(acc)
+        uint64_t pos = 0;
+
+        NarIndexer(NarAccessor & acc, Source & source)
+            : acc(acc), source(source)
         { }
 
-        void createMember(const Path & path, NarMember member) {
+        void createMember(const Path & path, NarMember member)
+        {
             size_t level = std::count(path.begin(), path.end(), '/');
             while (parents.size() > level) parents.pop();
 
@@ -77,33 +80,40 @@ struct NarAccessor : public FSAccessor
             parents.top()->isExecutable = true;
         }
 
-        void preallocateContents(unsigned long long size) override
+        void preallocateContents(uint64_t size) override
         {
-            currentStart = string(s, pos, 16);
-            assert(size <= std::numeric_limits<size_t>::max());
-            parents.top()->size = (size_t)size;
+            assert(size <= std::numeric_limits<uint64_t>::max());
+            parents.top()->size = (uint64_t) size;
             parents.top()->start = pos;
         }
 
-        void receiveContents(unsigned char * data, unsigned int len) override
-        {
-            // Sanity check
-            if (!currentStart.empty()) {
-                assert(len < 16 || currentStart == string((char *) data, 16));
-                currentStart.clear();
-            }
-        }
+        void receiveContents(unsigned char * data, size_t len) override
+        { }
 
         void createSymlink(const Path & path, const string & target) override
         {
             createMember(path,
                 NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
         }
+
+        size_t read(unsigned char * data, size_t len) override
+        {
+            auto n = source.read(data, len);
+            pos += n;
+            return n;
+        }
     };
 
     NarAccessor(ref<const std::string> nar) : nar(nar)
     {
-        NarIndexer indexer(*this, *nar);
+        StringSource source(*nar);
+        NarIndexer indexer(*this, source);
+        parseDump(indexer, indexer);
+    }
+
+    NarAccessor(Source & source)
+    {
+        NarIndexer indexer(*this, source);
         parseDump(indexer, indexer);
     }
 
@@ -219,6 +229,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
     return make_ref<NarAccessor>(nar);
 }
 
+ref<FSAccessor> makeNarAccessor(Source & source)
+{
+    return make_ref<NarAccessor>(source);
+}
+
 ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
     GetNarBytes getNarBytes)
 {

src/libstore/nar-accessor.hh

@@ -6,10 +6,14 @@
 
 namespace nix {
 
+struct Source;
+
 /* Return an object that provides access to the contents of a NAR
    file. */
 ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
 
+ref<FSAccessor> makeNarAccessor(Source & source);
+
 /* Create a NAR accessor from a NAR listing (in the format produced by
    listNar()). The callback getNarBytes(offset, length) is used by the
    readFile() method of the accessor to get the contents of files

src/libstore/nar-info-disk-cache.cc

@@ -189,13 +189,14 @@ public:
                 return {oInvalid, 0};
 
             auto namePart = queryNAR.getStr(1);
-            auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart));
+            auto narInfo = make_ref<NarInfo>(
+                StorePath(hashPart + "-" + namePart),
+                Hash::parseAnyPrefixed(queryNAR.getStr(6)));
             narInfo->url = queryNAR.getStr(2);
             narInfo->compression = queryNAR.getStr(3);
             if (!queryNAR.isNull(4))
-                narInfo->fileHash = Hash(queryNAR.getStr(4));
+                narInfo->fileHash = Hash::parseAnyPrefixed(queryNAR.getStr(4));
             narInfo->fileSize = queryNAR.getInt(5);
-            narInfo->narHash = Hash(queryNAR.getStr(6));
             narInfo->narSize = queryNAR.getInt(7);
             for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
                 narInfo->references.insert(StorePath(r));
@@ -230,7 +231,7 @@ public:
                     (std::string(info->path.name()))
                     (narInfo ? narInfo->url : "", narInfo != 0)
                     (narInfo ? narInfo->compression : "", narInfo != 0)
-                    (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash)
+                    (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash)
                     (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
                     (info->narHash.to_string(Base32, true))
                     (info->narSize)

src/libstore/nar-info.cc

@@ -1,36 +1,37 @@
 #include "globals.hh"
 #include "nar-info.hh"
+#include "store-api.hh"
 
 namespace nix {
 
 NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
-    : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack
+    : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack
 {
     auto corrupt = [&]() {
-        throw Error("NAR info file '%1%' is corrupt", whence);
+        return Error("NAR info file '%1%' is corrupt", whence);
     };
 
     auto parseHashField = [&](const string & s) {
         try {
-            return Hash(s);
+            return Hash::parseAnyPrefixed(s);
         } catch (BadHash &) {
-            corrupt();
-            return Hash(); // never reached
+            throw corrupt();
         }
     };
 
     bool havePath = false;
+    bool haveNarHash = false;
 
     size_t pos = 0;
     while (pos < s.size()) {
 
         size_t colon = s.find(':', pos);
-        if (colon == std::string::npos) corrupt();
+        if (colon == std::string::npos) throw corrupt();
 
         std::string name(s, pos, colon - pos);
 
         size_t eol = s.find('\n', colon + 2);
-        if (eol == std::string::npos) corrupt();
+        if (eol == std::string::npos) throw corrupt();
 
         std::string value(s, colon + 2, eol - colon - 2);
 
@@ -45,16 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
         else if (name == "FileHash")
             fileHash = parseHashField(value);
         else if (name == "FileSize") {
-            if (!string2Int(value, fileSize)) corrupt();
+            if (!string2Int(value, fileSize)) throw corrupt();
         }
-        else if (name == "NarHash")
+        else if (name == "NarHash") {
             narHash = parseHashField(value);
+            haveNarHash = true;
+        }
         else if (name == "NarSize") {
-            if (!string2Int(value, narSize)) corrupt();
+            if (!string2Int(value, narSize)) throw corrupt();
         }
         else if (name == "References") {
             auto refs = tokenizeString<Strings>(value, " ");
-            if (!references.empty()) corrupt();
+            if (!references.empty()) throw corrupt();
             for (auto & r : refs)
                 references.insert(StorePath(r));
         }
@@ -67,7 +70,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
         else if (name == "Sig")
             sigs.insert(value);
         else if (name == "CA") {
-            if (ca) corrupt();
+            if (ca) throw corrupt();
             // FIXME: allow blank ca or require skipping field?
             ca = parseContentAddressOpt(value);
         }
@@ -77,7 +80,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
 
     if (compression == "") compression = "bzip2";
 
-    if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt();
+    if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt();
 }
 
 std::string NarInfo::to_string(const Store & store) const
@@ -87,8 +90,8 @@ std::string NarInfo::to_string(const Store & store) const
     res += "URL: " + url + "\n";
     assert(compression != "");
     res += "Compression: " + compression + "\n";
-    assert(fileHash.type == htSHA256);
-    res += "FileHash: " + fileHash.to_string(Base32, true) + "\n";
+    assert(fileHash && fileHash->type == htSHA256);
+    res += "FileHash: " + fileHash->to_string(Base32, true) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
     assert(narHash.type == htSHA256);
     res += "NarHash: " + narHash.to_string(Base32, true) + "\n";

src/libstore/nar-info.hh

@@ -2,20 +2,22 @@
 
 #include "types.hh"
 #include "hash.hh"
-#include "store-api.hh"
+#include "path-info.hh"
 
 namespace nix {
 
+class Store;
+
 struct NarInfo : ValidPathInfo
 {
     std::string url;
     std::string compression;
-    Hash fileHash;
+    std::optional<Hash> fileHash;
     uint64_t fileSize = 0;
     std::string system;
 
     NarInfo() = delete;
-    NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { }
+    NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { }
     NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
     NarInfo(const Store & store, const std::string & s, const std::string & whence);
 

src/libstore/optimise-store.cc

@@ -17,9 +17,7 @@ namespace nix {
 
 static void makeWritable(const Path & path)
 {
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
+    auto st = lstat(path);
     if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
         throw SysError("changing writability of '%1%'", path);
 }
@@ -94,9 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
 {
     checkInterrupt();
 
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError("getting attributes of path '%1%'", path);
+    auto st = lstat(path);
 
 #if __APPLE__
     /* HFS/macOS has some undocumented security feature disabling hardlinking for
@@ -187,9 +183,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
 
     /* Yes!  We've seen a file with the same contents.  Replace the
        current file with a hard link to that file. */
-    struct stat stLink;
-    if (lstat(linkPath.c_str(), &stLink))
-        throw SysError("getting attributes of path '%1%'", linkPath);
+    auto stLink = lstat(linkPath);
 
     if (st.st_ino == stLink.st_ino) {
         debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
@@ -282,21 +276,15 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
     }
 }
 
-static string showBytes(unsigned long long bytes)
-{
-    return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
-}
-
 void LocalStore::optimiseStore()
 {
     OptimiseStats stats;
 
     optimiseStore(stats);
 
-    printInfo(
-        format("%1% freed by hard-linking %2% files")
-        % showBytes(stats.bytesFreed)
-        % stats.filesLinked);
+    printInfo("%s freed by hard-linking %d files",
+        showBytes(stats.bytesFreed),
+        stats.filesLinked);
 }
 
 void LocalStore::optimisePath(const Path & path)

src/libstore/parsed-derivations.cc

@@ -94,7 +94,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
     return res;
 }
 
-bool ParsedDerivation::canBuildLocally() const
+bool ParsedDerivation::canBuildLocally(Store & localStore) const
 {
     if (drv.platform != settings.thisSystem.get()
         && !settings.extraPlatforms.get().count(drv.platform)
@@ -102,14 +102,14 @@ bool ParsedDerivation::canBuildLocally() const
         return false;
 
     for (auto & feature : getRequiredSystemFeatures())
-        if (!settings.systemFeatures.get().count(feature)) return false;
+        if (!localStore.systemFeatures.get().count(feature)) return false;
 
     return true;
 }
 
-bool ParsedDerivation::willBuildLocally() const
+bool ParsedDerivation::willBuildLocally(Store & localStore) const
 {
-    return getBoolAttr("preferLocalBuild") && canBuildLocally();
+    return getBoolAttr("preferLocalBuild") && canBuildLocally(localStore);
 }
 
 bool ParsedDerivation::substitutesAllowed() const
@@ -117,9 +117,4 @@ bool ParsedDerivation::substitutesAllowed() const
     return getBoolAttr("allowSubstitutes", true);
 }
 
-bool ParsedDerivation::contentAddressed() const
-{
-    return getBoolAttr("__contentAddressed", false);
-}
-
 }

src/libstore/parsed-derivations.hh

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "store-api.hh"
 
 #include <nlohmann/json_fwd.hpp>
@@ -29,13 +31,11 @@ public:
 
     StringSet getRequiredSystemFeatures() const;
 
-    bool canBuildLocally() const;
+    bool canBuildLocally(Store & localStore) const;
 
-    bool willBuildLocally() const;
+    bool willBuildLocally(Store & localStore) const;
 
     bool substitutesAllowed() const;
-
-    bool contentAddressed() const;
 };
 
 }

src/libstore/path-info.hh

@@ -0,0 +1,112 @@
+#pragma once
+
+#include "crypto.hh"
+#include "path.hh"
+#include "hash.hh"
+#include "content-address.hh"
+
+#include <string>
+#include <optional>
+
+namespace nix {
+
+
+class Store;
+
+
+struct SubstitutablePathInfo
+{
+    std::optional<StorePath> deriver;
+    StorePathSet references;
+    uint64_t downloadSize; /* 0 = unknown or inapplicable */
+    uint64_t narSize; /* 0 = unknown */
+};
+
+typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
+
+
+struct ValidPathInfo
+{
+    StorePath path;
+    std::optional<StorePath> deriver;
+    // TODO document this
+    Hash narHash;
+    StorePathSet references;
+    time_t registrationTime = 0;
+    uint64_t narSize = 0; // 0 = unknown
+    uint64_t id; // internal use only
+
+    /* Whether the path is ultimately trusted, that is, it's a
+       derivation output that was built locally. */
+    bool ultimate = false;
+
+    StringSet sigs; // note: not necessarily verified
+
+    /* If non-empty, an assertion that the path is content-addressed,
+       i.e., that the store path is computed from a cryptographic hash
+       of the contents of the path, plus some other bits of data like
+       the "name" part of the path. Such a path doesn't need
+       signatures, since we don't have to trust anybody's claim that
+       the path is the output of a particular derivation. (In the
+       extensional store model, we have to trust that the *contents*
+       of an output path of a derivation were actually produced by
+       that derivation. In the intensional model, we have to trust
+       that a particular output path was produced by a derivation; the
+       path then implies the contents.)
+
+       Ideally, the content-addressability assertion would just be a Boolean,
+       and the store path would be computed from the name component, ‘narHash’
+       and ‘references’. However, we support many types of content addresses.
+    */
+    std::optional<ContentAddress> ca;
+
+    bool operator == (const ValidPathInfo & i) const
+    {
+        return
+            path == i.path
+            && narHash == i.narHash
+            && references == i.references;
+    }
+
+    /* Return a fingerprint of the store path to be used in binary
+       cache signatures. It contains the store path, the base-32
+       SHA-256 hash of the NAR serialisation of the path, the size of
+       the NAR, and the sorted references. The size field is strictly
+       speaking superfluous, but might prevent endless/excessive data
+       attacks. */
+    std::string fingerprint(const Store & store) const;
+
+    void sign(const Store & store, const SecretKey & secretKey);
+
+    /* Return true iff the path is verifiably content-addressed. */
+    bool isContentAddressed(const Store & store) const;
+
+    /* Functions to view references + hasSelfReference as one set, mainly for
+       compatibility's sake. */
+    StorePathSet referencesPossiblyToSelf() const;
+    void insertReferencePossiblyToSelf(StorePath && ref);
+    void setReferencesPossiblyToSelf(StorePathSet && refs);
+
+    static const size_t maxSigs = std::numeric_limits<size_t>::max();
+
+    /* Return the number of signatures on this .narinfo that were
+       produced by one of the specified keys, or maxSigs if the path
+       is content-addressed. */
+    size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
+
+    /* Verify a single signature. */
+    bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
+
+    Strings shortRefs() const;
+
+    ValidPathInfo(const ValidPathInfo & other) = default;
+
+    ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
+    ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
+
+    virtual ~ValidPathInfo() { }
+};
+
+typedef list<ValidPathInfo> ValidPathInfos;
+
+}

src/libstore/path.cc

@@ -2,8 +2,6 @@
 
 namespace nix {
 
-MakeError(BadStorePath, Error);
-
 static void checkName(std::string_view path, std::string_view name)
 {
     if (name.empty())

src/libstore/path.hh

@@ -64,6 +64,8 @@ typedef std::set<StorePath> StorePathSet;
 typedef std::vector<StorePath> StorePaths;
 typedef std::map<string, StorePath> OutputPathMap;
 
+typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
+
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 

src/libstore/profiles.cc

@@ -12,30 +12,24 @@
 namespace nix {
 
 
-static bool cmpGensByNumber(const Generation & a, const Generation & b)
-{
-    return a.number < b.number;
-}
-
-
 /* Parse a generation name of the format
    `<profilename>-<number>-link'. */
-static int parseName(const string & profileName, const string & name)
+static std::optional<GenerationNumber> parseName(const string & profileName, const string & name)
 {
-    if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1;
+    if (string(name, 0, profileName.size() + 1) != profileName + "-") return {};
     string s = string(name, profileName.size() + 1);
     string::size_type p = s.find("-link");
-    if (p == string::npos) return -1;
-    int n;
+    if (p == string::npos) return {};
+    unsigned int n;
     if (string2Int(string(s, 0, p), n) && n >= 0)
         return n;
     else
-        return -1;
+        return {};
 }
 
 
 
-Generations findGenerations(Path profile, int & curGen)
+std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile)
 {
     Generations gens;
 
@@ -43,30 +37,31 @@ Generations findGenerations(Path profile, int & curGen)
     auto profileName = std::string(baseNameOf(profile));
 
     for (auto & i : readDirectory(profileDir)) {
-        int n;
-        if ((n = parseName(profileName, i.name)) != -1) {
-            Generation gen;
-            gen.path = profileDir + "/" + i.name;
-            gen.number = n;
-            struct stat st;
-            if (lstat(gen.path.c_str(), &st) != 0)
-                throw SysError("statting '%1%'", gen.path);
-            gen.creationTime = st.st_mtime;
-            gens.push_back(gen);
+        if (auto n = parseName(profileName, i.name)) {
+            auto path = profileDir + "/" + i.name;
+            gens.push_back({
+                .number = *n,
+                .path = path,
+                .creationTime = lstat(path).st_mtime
+            });
         }
     }
 
-    gens.sort(cmpGensByNumber);
+    gens.sort([](const Generation & a, const Generation & b)
+    {
+        return a.number < b.number;
+    });
 
-    curGen = pathExists(profile)
+    return {
+        gens,
+        pathExists(profile)
         ? parseName(profileName, readLink(profile))
-        : -1;
-
-    return gens;
+        : std::nullopt
+    };
 }
 
 
-static void makeName(const Path & profile, unsigned int num,
+static void makeName(const Path & profile, GenerationNumber num,
     Path & outLink)
 {
     Path prefix = (format("%1%-%2%") % profile % num).str();
@@ -74,18 +69,17 @@ static void makeName(const Path & profile, unsigned int num,
 }
 
 
-Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
+Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
 {
     /* The new generation number should be higher than old the
        previous ones. */
-    int dummy;
-    Generations gens = findGenerations(profile, dummy);
+    auto [gens, dummy] = findGenerations(profile);
 
-    unsigned int num;
+    GenerationNumber num;
     if (gens.size() > 0) {
         Generation last = gens.back();
 
-        if (readLink(last.path) == outPath) {
+        if (readLink(last.path) == store->printStorePath(outPath)) {
             /* We only create a new generation symlink if it differs
                from the last one.
 
@@ -108,7 +102,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
        user environment etc. we've just built. */
     Path generation;
     makeName(profile, num + 1, generation);
-    store->addPermRoot(store->parseStorePath(outPath), generation, false, true);
+    store->addPermRoot(outPath, generation);
 
     return generation;
 }
@@ -121,7 +115,7 @@ static void removeFile(const Path & path)
 }
 
 
-void deleteGeneration(const Path & profile, unsigned int gen)
+void deleteGeneration(const Path & profile, GenerationNumber gen)
 {
     Path generation;
     makeName(profile, gen, generation);
@@ -129,7 +123,7 @@ void deleteGeneration(const Path & profile, unsigned int gen)
 }
 
 
-static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun)
+static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun)
 {
     if (dryRun)
         printInfo(format("would remove generation %1%") % gen);
@@ -140,31 +134,29 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu
 }
 
 
-void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun)
+void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun)
 {
     PathLocks lock;
     lockProfile(lock, profile);
 
-    int curGen;
-    Generations gens = findGenerations(profile, curGen);
+    auto [gens, curGen] = findGenerations(profile);
 
-    if (gensToDelete.find(curGen) != gensToDelete.end())
+    if (gensToDelete.count(*curGen))
         throw Error("cannot delete current generation of profile %1%'", profile);
 
     for (auto & i : gens) {
-        if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
+        if (!gensToDelete.count(i.number)) continue;
         deleteGeneration2(profile, i.number, dryRun);
     }
 }
 
-void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
+void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun)
 {
     PathLocks lock;
     lockProfile(lock, profile);
 
-    int curGen;
     bool fromCurGen = false;
-    Generations gens = findGenerations(profile, curGen);
+    auto [gens, curGen] = findGenerations(profile);
     for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
         if (i->number == curGen) {
             fromCurGen = true;
@@ -186,8 +178,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun)
     PathLocks lock;
     lockProfile(lock, profile);
 
-    int curGen;
-    Generations gens = findGenerations(profile, curGen);
+    auto [gens, curGen] = findGenerations(profile);
 
     for (auto & i : gens)
         if (i.number != curGen)
@@ -200,8 +191,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
     PathLocks lock;
     lockProfile(lock, profile);
 
-    int curGen;
-    Generations gens = findGenerations(profile, curGen);
+    auto [gens, curGen] = findGenerations(profile);
 
     bool canDelete = false;
     for (auto i = gens.rbegin(); i != gens.rend(); ++i)

src/libstore/profiles.hh

@@ -8,38 +8,35 @@
 
 namespace nix {
 
+class StorePath;
+
+
+typedef unsigned int GenerationNumber;
 
 struct Generation
 {
-    int number;
+    GenerationNumber number;
     Path path;
     time_t creationTime;
-    Generation()
-    {
-        number = -1;
-    }
-    operator bool() const
-    {
-        return number != -1;
-    }
 };
 
-typedef list<Generation> Generations;
+typedef std::list<Generation> Generations;
 
 
 /* Returns the list of currently present generations for the specified
-   profile, sorted by generation number. */
-Generations findGenerations(Path profile, int & curGen);
+   profile, sorted by generation number. Also returns the number of
+   the current generation. */
+std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
 
 class LocalFSStore;
 
-Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath);
+Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath);
 
-void deleteGeneration(const Path & profile, unsigned int gen);
+void deleteGeneration(const Path & profile, GenerationNumber gen);
 
-void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun);
+void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun);
 
-void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun);
+void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun);
 
 void deleteOldGenerations(const Path & profile, bool dryRun);
 

src/libstore/references.cc

@@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len,
 
 struct RefScanSink : Sink
 {
-    HashSink hashSink;
     StringSet hashes;
     StringSet seen;
 
     string tail;
 
-    RefScanSink() : hashSink(htSHA256) { }
+    RefScanSink() { }
 
     void operator () (const unsigned char * data, size_t len);
 };
@@ -62,8 +61,6 @@ struct RefScanSink : Sink
 
 void RefScanSink::operator () (const unsigned char * data, size_t len)
 {
-    hashSink(data, len);
-
     /* It's possible that a reference spans the previous and current
        fragment, so search in the concatenation of the tail of the
        previous fragment and the start of the current fragment. */
@@ -79,10 +76,20 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
 }
 
 
-PathSet scanForReferences(const string & path,
-    const PathSet & refs, HashResult & hash)
+std::pair<PathSet, HashResult> scanForReferences(const string & path,
+    const PathSet & refs)
 {
-    RefScanSink sink;
+    HashSink hashSink { htSHA256 };
+    auto found = scanForReferences(hashSink, path, refs);
+    auto hash = hashSink.finish();
+    return std::pair<PathSet, HashResult>(found, hash);
+}
+
+PathSet scanForReferences(Sink & toTee,
+    const string & path, const PathSet & refs)
+{
+    RefScanSink refsSink;
+    TeeSink sink { refsSink, toTee };
     std::map<string, Path> backMap;
 
     /* For efficiency (and a higher hit rate), just search for the
@@ -97,7 +104,7 @@ PathSet scanForReferences(const string & path,
         assert(s.size() == refLength);
         assert(backMap.find(s) == backMap.end());
         // parseHash(htSHA256, s);
-        sink.hashes.insert(s);
+        refsSink.hashes.insert(s);
         backMap[s] = i;
     }
 
@@ -106,14 +113,12 @@ PathSet scanForReferences(const string & path,
 
     /* Map the hashes found back to their store paths. */
     PathSet found;
-    for (auto & i : sink.seen) {
+    for (auto & i : refsSink.seen) {
         std::map<string, Path>::iterator j;
         if ((j = backMap.find(i)) == backMap.end()) abort();
         found.insert(j->second);
     }
 
-    hash = sink.hashSink.finish();
-
     return found;
 }
 

src/libstore/references.hh

@@ -5,8 +5,9 @@
 
 namespace nix {
 
-PathSet scanForReferences(const Path & path, const PathSet & refs,
-    HashResult & hash);
+std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
+
+PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
 
 struct RewritingSink : Sink
 {

src/libstore/remote-fs-accessor.cc

@@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
         createDirs(cacheDir);
 }
 
-Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
+Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
 {
     assert(cacheDir != "");
-    return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
+    return fmt("%s/%s.%s", cacheDir, hashPart, ext);
 }
 
-void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
+void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
     ref<FSAccessor> narAccessor)
 {
-    nars.emplace(storePath, narAccessor);
+    nars.emplace(hashPart, narAccessor);
 
     if (cacheDir != "") {
         try {
             std::ostringstream str;
             JSONPlaceholder jsonRoot(str);
             listNar(jsonRoot, narAccessor, "", true);
-            writeFile(makeCacheFile(storePath, "ls"), str.str());
+            writeFile(makeCacheFile(hashPart, "ls"), str.str());
 
             /* FIXME: do this asynchronously. */
-            writeFile(makeCacheFile(storePath, "nar"), nar);
+            writeFile(makeCacheFile(hashPart, "nar"), nar);
 
         } catch (...) {
             ignoreException();
@@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
 {
     auto path = canonPath(path_);
 
-    auto storePath = store->toStorePath(path);
-    std::string restPath = std::string(path, storePath.size());
+    auto [storePath, restPath] = store->toStorePath(path);
 
-    if (!store->isValidPath(store->parseStorePath(storePath)))
-        throw InvalidPath("path '%1%' is not a valid store path", storePath);
+    if (!store->isValidPath(storePath))
+        throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
 
-    auto i = nars.find(storePath);
+    auto i = nars.find(std::string(storePath.hashPart()));
     if (i != nars.end()) return {i->second, restPath};
 
     StringSink sink;
     std::string listing;
     Path cacheFile;
 
-    if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
+    if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
 
         try {
-            listing = nix::readFile(makeCacheFile(storePath, "ls"));
+            listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
 
             auto narAccessor = makeLazyNarAccessor(listing,
                 [cacheFile](uint64_t offset, uint64_t length) {
@@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
                     return buf;
                 });
 
-            nars.emplace(storePath, narAccessor);
+            nars.emplace(storePath.hashPart(), narAccessor);
             return {narAccessor, restPath};
 
         } catch (SysError &) { }
@@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
             *sink.s = nix::readFile(cacheFile);
 
             auto narAccessor = makeNarAccessor(sink.s);
-            nars.emplace(storePath, narAccessor);
+            nars.emplace(storePath.hashPart(), narAccessor);
             return {narAccessor, restPath};
 
         } catch (SysError &) { }
     }
 
-    store->narFromPath(store->parseStorePath(storePath), sink);
+    store->narFromPath(storePath, sink);
     auto narAccessor = makeNarAccessor(sink.s);
-    addToCache(storePath, *sink.s, narAccessor);
+    addToCache(storePath.hashPart(), *sink.s, narAccessor);
     return {narAccessor, restPath};
 }
 

src/libstore/remote-fs-accessor.hh

@@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor
 {
     ref<Store> store;
 
-    std::map<Path, ref<FSAccessor>> nars;
+    std::map<std::string, ref<FSAccessor>> nars;
 
     Path cacheDir;
 
@@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor
 
     friend class BinaryCacheStore;
 
-    Path makeCacheFile(const Path & storePath, const std::string & ext);
+    Path makeCacheFile(std::string_view hashPart, const std::string & ext);
 
-    void addToCache(const Path & storePath, const std::string & nar,
+    void addToCache(std::string_view hashPart, const std::string & nar,
         ref<FSAccessor> narAccessor);
 
 public:

src/libstore/remote-store.cc

@@ -1,5 +1,6 @@
 #include "serialise.hh"
 #include "util.hh"
+#include "remote-fs-accessor.hh"
 #include "remote-store.hh"
 #include "worker-protocol.hh"
 #include "archive.hh"
@@ -9,6 +10,7 @@
 #include "pool.hh"
 #include "finally.hh"
 #include "logging.hh"
+#include "callback.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -22,53 +24,82 @@
 
 namespace nix {
 
+namespace worker_proto {
 
-template<> StorePathSet readStorePaths(const Store & store, Source & from)
+std::string read(const Store & store, Source & from, Phantom<std::string> _)
 {
-    StorePathSet paths;
-    for (auto & i : readStrings<Strings>(from))
-        paths.insert(store.parseStorePath(i));
-    return paths;
+    return readString(from);
+}
+
+void write(const Store & store, Sink & out, const std::string & str)
+{
+    out << str;
 }
 
 
-void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths)
+StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
 {
-    out << paths.size();
-    for (auto & i : paths)
-        out << store.printStorePath(i);
+    return store.parseStorePath(readString(from));
 }
 
-std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
+void write(const Store & store, Sink & out, const StorePath & storePath)
 {
-    std::map<string, StorePath> pathMap;
-    auto rawInput = readStrings<Strings>(from);
-    if (rawInput.size() % 2)
-        throw Error("got an odd number of elements from the daemon when trying to read a output path map");
-    auto curInput = rawInput.begin();
-    while (curInput != rawInput.end()) {
-        auto thisKey = *curInput++;
-        auto thisValue = *curInput++;
-        pathMap.emplace(thisKey, store.parseStorePath(thisValue));
-    }
-    return pathMap;
+    out << store.printStorePath(storePath);
 }
 
-void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap)
+
+ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
 {
-    out << 2*pathMap.size();
-    for (auto & i : pathMap) {
-        out << i.first;
-        out << store.printStorePath(i.second);
-    }
+    return parseContentAddress(readString(from));
 }
 
+void write(const Store & store, Sink & out, const ContentAddress & ca)
+{
+    out << renderContentAddress(ca);
+}
+
+
+std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
+{
+    auto s = readString(from);
+    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
+}
+
+void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
+{
+    out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
+}
+
+
+std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
+{
+    return parseContentAddressOpt(readString(from));
+}
+
+void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
+{
+    out << (caOpt ? renderContentAddress(*caOpt) : "");
+}
+
+}
+
+
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : Store(params)
+    , RemoteStoreConfig(params)
     , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
-            [this]() { return openConnectionWrapper(); },
+            [this]() {
+                auto conn = openConnectionWrapper();
+                try {
+                    initConnection(*conn);
+                } catch (...) {
+                    failed = true;
+                    throw;
+                }
+                return conn;
+            },
             [this](const ref<Connection> & r) {
                 return
                     r->to.good()
@@ -95,19 +126,21 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
 
 
 UDSRemoteStore::UDSRemoteStore(const Params & params)
-    : Store(params)
+    : StoreConfig(params)
+    , Store(params)
     , LocalFSStore(params)
     , RemoteStore(params)
 {
 }
 
 
-UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
-    : Store(params)
-    , LocalFSStore(params)
-    , RemoteStore(params)
-    , path(socket_path)
+UDSRemoteStore::UDSRemoteStore(
+        const std::string scheme,
+        std::string socket_path,
+        const Params & params)
+    : UDSRemoteStore(params)
 {
+    path.emplace(socket_path);
 }
 
 
@@ -151,8 +184,6 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
 
     conn->startTime = std::chrono::steady_clock::now();
 
-    initConnection(*conn);
-
     return conn;
 }
 
@@ -260,14 +291,16 @@ struct ConnectionHandle
 
     RemoteStore::Connection * operator -> () { return &*handle; }
 
-    void processStderr(Sink * sink = 0, Source * source = 0)
+    void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
     {
-        auto ex = handle->processStderr(sink, source);
+        auto ex = handle->processStderr(sink, source, flush);
         if (ex) {
             daemonException = true;
             std::rethrow_exception(ex);
         }
     }
+
+    void withFramedSink(std::function<void(Sink & sink)> fun);
 };
 
 
@@ -296,9 +329,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         return res;
     } else {
         conn->to << wopQueryValidPaths;
-        writeStorePaths(*this, conn->to, paths);
+        worker_proto::write(*this, conn->to, paths);
         conn.processStderr();
-        return readStorePaths<StorePathSet>(*this, conn->from);
+        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
 }
 
@@ -308,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
     auto conn(getConnection());
     conn->to << wopQueryAllValidPaths;
     conn.processStderr();
-    return readStorePaths<StorePathSet>(*this, conn->from);
+    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
 }
 
 
@@ -325,41 +358,46 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
         return res;
     } else {
         conn->to << wopQuerySubstitutablePaths;
-        writeStorePaths(*this, conn->to, paths);
+        worker_proto::write(*this, conn->to, paths);
         conn.processStderr();
-        return readStorePaths<StorePathSet>(*this, conn->from);
+        return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
     }
 }
 
 
-void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
-    SubstitutablePathInfos & infos)
+void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
 {
-    if (paths.empty()) return;
+    if (pathsMap.empty()) return;
 
     auto conn(getConnection());
 
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
 
-        for (auto & i : paths) {
+        for (auto & i : pathsMap) {
             SubstitutablePathInfo info;
-            conn->to << wopQuerySubstitutablePathInfo << printStorePath(i);
+            conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
             conn.processStderr();
             unsigned int reply = readInt(conn->from);
             if (reply == 0) continue;
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = readStorePaths<StorePathSet>(*this, conn->from);
+            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
-            infos.insert_or_assign(i, std::move(info));
+            infos.insert_or_assign(i.first, std::move(info));
         }
 
     } else {
 
         conn->to << wopQuerySubstitutablePathInfos;
-        writeStorePaths(*this, conn->to, paths);
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
+            StorePathSet paths;
+            for (auto & path : pathsMap)
+                paths.insert(path.first);
+            worker_proto::write(*this, conn->to, paths);
+        } else
+            worker_proto::write(*this, conn->to, pathsMap);
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
@@ -367,7 +405,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references = readStorePaths<StorePathSet>(*this, conn->from);
+            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -376,11 +414,28 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
 }
 
 
+ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path)
+{
+    auto deriver = readString(conn->from);
+    auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
+    auto info = make_ref<ValidPathInfo>(path, narHash);
+    if (deriver != "") info->deriver = parseStorePath(deriver);
+    info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+    conn->from >> info->registrationTime >> info->narSize;
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
+        conn->from >> info->ultimate;
+        info->sigs = readStrings<StringSet>(conn->from);
+        info->ca = parseContentAddressOpt(readString(conn->from));
+    }
+    return info;
+}
+
+
 void RemoteStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        std::shared_ptr<ValidPathInfo> info;
+        std::shared_ptr<const ValidPathInfo> info;
         {
             auto conn(getConnection());
             conn->to << wopQueryPathInfo << printStorePath(path);
@@ -396,17 +451,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
                 bool valid; conn->from >> valid;
                 if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
             }
-            info = std::make_shared<ValidPathInfo>(StorePath(path));
-            auto deriver = readString(conn->from);
-            if (deriver != "") info->deriver = parseStorePath(deriver);
-            info->narHash = Hash(readString(conn->from), htSHA256);
-            info->references = readStorePaths<StorePathSet>(*this, conn->from);
-            conn->from >> info->registrationTime >> info->narSize;
-            if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
-                conn->from >> info->ultimate;
-                info->sigs = readStrings<StringSet>(conn->from);
-                info->ca = parseContentAddressOpt(readString(conn->from));
-            }
+            info = readValidPathInfo(conn, path);
         }
         callback(std::move(info));
     } catch (...) { callback.rethrow(); }
@@ -419,7 +464,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
     auto conn(getConnection());
     conn->to << wopQueryReferrers << printStorePath(path);
     conn.processStderr();
-    for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
+    for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
         referrers.insert(i);
 }
 
@@ -429,7 +474,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
     auto conn(getConnection());
     conn->to << wopQueryValidDerivers << printStorePath(path);
     conn.processStderr();
-    return readStorePaths<StorePathSet>(*this, conn->from);
+    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
 }
 
 
@@ -441,17 +486,32 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
     }
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
-    return readStorePaths<StorePathSet>(*this, conn->from);
+    return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
 }
 
 
-OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
 {
-    auto conn(getConnection());
-    conn->to << wopQueryDerivationOutputMap << printStorePath(path);
-    conn.processStderr();
-    return readOutputPathMap(*this, conn->from);
-
+    if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
+        auto conn(getConnection());
+        conn->to << wopQueryDerivationOutputMap << printStorePath(path);
+        conn.processStderr();
+        return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
+    } else {
+        // Fallback for old daemon versions.
+        // For floating-CA derivations (and their co-dependencies) this is an
+        // under-approximation as it only returns the paths that can be inferred
+        // from the derivation itself (and not the ones that are known because
+        // the have been built), but as old stores don't handle floating-CA
+        // derivations this shouldn't matter
+        auto derivation = readDerivation(path);
+        auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this);
+        std::map<std::string, std::optional<StorePath>> ret;
+        for (auto & [outputName, outputAndPath] : outputsWithOptPaths) {
+            ret.emplace(outputName, outputAndPath.second);
+        }
+        return ret;
+    }
 }
 
 std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
@@ -465,8 +525,95 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 }
 
 
+ref<const ValidPathInfo> RemoteStore::addCAToStore(
+    Source & dump,
+    const string & name,
+    ContentAddressMethod caMethod,
+    const StorePathSet & references,
+    RepairFlag repair)
+{
+    std::optional<ConnectionHandle> conn_(getConnection());
+    auto & conn = *conn_;
+
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
+
+        conn->to
+            << wopAddToStore
+            << name
+            << renderContentAddressMethod(caMethod);
+        worker_proto::write(*this, conn->to, references);
+        conn->to << repair;
+
+        conn.withFramedSink([&](Sink & sink) {
+            dump.drainInto(sink);
+        });
+
+        auto path = parseStorePath(readString(conn->from));
+        return readValidPathInfo(conn, path);
+    }
+    else {
+        if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
+
+        std::visit(overloaded {
+            [&](TextHashMethod thm) -> void {
+                std::string s = dump.drain();
+                conn->to << wopAddTextToStore << name << s;
+                worker_proto::write(*this, conn->to, references);
+                conn.processStderr();
+            },
+            [&](FixedOutputHashMethod fohm) -> void {
+                conn->to
+                    << wopAddToStore
+                    << name
+                    << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+                    << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
+                    << printHashType(fohm.hashType);
+
+                try {
+                    conn->to.written = 0;
+                    conn->to.warn = true;
+                    connections->incCapacity();
+                    {
+                        Finally cleanup([&]() { connections->decCapacity(); });
+                        if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
+                            dump.drainInto(conn->to);
+                        } else {
+                            std::string contents = dump.drain();
+                            dumpString(contents, conn->to);
+                        }
+                    }
+                    conn->to.warn = false;
+                    conn.processStderr();
+                } catch (SysError & e) {
+                    /* Daemon closed while we were sending the path. Probably OOM
+                      or I/O error. */
+                    if (e.errNo == EPIPE)
+                        try {
+                            conn.processStderr();
+                        } catch (EndOfFile & e) { }
+                    throw;
+                }
+
+            }
+        }, caMethod);
+        auto path = parseStorePath(readString(conn->from));
+        // Release our connection to prevent a deadlock in queryPathInfo().
+        conn_.reset();
+        return queryPathInfo(path);
+    }
+}
+
+
+StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method, HashType hashType, RepairFlag repair)
+{
+    StorePathSet references;
+    return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
+}
+
+
 void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
-    RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+    RepairFlag repair, CheckSigsFlag checkSigs)
 {
     auto conn(getConnection());
 
@@ -480,7 +627,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             sink
                 << exportMagic
                 << printStorePath(info.path);
-            writeStorePaths(*this, sink, info.references);
+            worker_proto::write(*this, sink, info.references);
             sink
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
@@ -490,7 +637,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
         conn.processStderr(0, source2.get());
 
-        auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from);
+        auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
         assert(importedPaths.size() <= 1);
     }
 
@@ -499,68 +646,30 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
                  << info.narHash.to_string(Base16, false);
-        writeStorePaths(*this, conn->to, info.references);
+        worker_proto::write(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
-        bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
-        if (!tunnel) copyNAR(source, conn->to);
-        conn.processStderr(0, tunnel ? &source : nullptr);
-    }
-}
 
-
-StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
-    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
-{
-    if (repair) throw Error("repairing is not supported when building through the Nix daemon");
-
-    auto conn(getConnection());
-
-    Path srcPath(absPath(_srcPath));
-
-    conn->to
-        << wopAddToStore
-        << name
-        << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
-        << (method == FileIngestionMethod::Recursive ? 1 : 0)
-        << printHashType(hashAlgo);
-
-    try {
-        conn->to.written = 0;
-        conn->to.warn = true;
-        connections->incCapacity();
-        {
-            Finally cleanup([&]() { connections->decCapacity(); });
-            dumpPath(srcPath, conn->to, filter);
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
+            conn.withFramedSink([&](Sink & sink) {
+                copyNAR(source, sink);
+            });
+        } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
+            conn.processStderr(0, &source);
+        } else {
+            copyNAR(source, conn->to);
+            conn.processStderr(0, nullptr);
         }
-        conn->to.warn = false;
-        conn.processStderr();
-    } catch (SysError & e) {
-        /* Daemon closed while we were sending the path. Probably OOM
-           or I/O error. */
-        if (e.errNo == EPIPE)
-            try {
-                conn.processStderr();
-            } catch (EndOfFile & e) { }
-        throw;
     }
-
-    return parseStorePath(readString(conn->from));
 }
 
 
 StorePath RemoteStore::addTextToStore(const string & name, const string & s,
     const StorePathSet & references, RepairFlag repair)
 {
-    if (repair) throw Error("repairing is not supported when building through the Nix daemon");
-
-    auto conn(getConnection());
-    conn->to << wopAddTextToStore << name << s;
-    writeStorePaths(*this, conn->to, references);
-
-    conn.processStderr();
-    return parseStorePath(readString(conn->from));
+    StringSource source(s);
+    return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
 }
 
 
@@ -659,7 +768,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     conn->to
         << wopCollectGarbage << options.action;
-    writeStorePaths(*this, conn->to, options.pathsToDelete);
+    worker_proto::write(*this, conn->to, options.pathsToDelete);
     conn->to << options.ignoreLiveness
         << options.maxFreed
         /* removed options */
@@ -707,7 +816,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
 
 void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
     StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-    unsigned long long & downloadSize, unsigned long long & narSize)
+    uint64_t & downloadSize, uint64_t & narSize)
 {
     {
         auto conn(getConnection());
@@ -721,9 +830,9 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
             ss.push_back(p.to_string(*this));
         conn->to << ss;
         conn.processStderr();
-        willBuild = readStorePaths<StorePathSet>(*this, conn->from);
-        willSubstitute = readStorePaths<StorePathSet>(*this, conn->from);
-        unknown = readStorePaths<StorePathSet>(*this, conn->from);
+        willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
+        unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
         conn->from >> downloadSize >> narSize;
         return;
     }
@@ -762,6 +871,18 @@ RemoteStore::Connection::~Connection()
     }
 }
 
+void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
+{
+    auto conn(connections->get());
+    conn->to << wopNarFromPath << printStorePath(path);
+    conn->processStderr();
+    copyNAR(conn->from, sink);
+}
+
+ref<FSAccessor> RemoteStore::getFSAccessor()
+{
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
+}
 
 static Logger::Fields readFields(Source & from)
 {
@@ -780,9 +901,10 @@ static Logger::Fields readFields(Source & from)
 }
 
 
-std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source)
+std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
 {
-    to.flush();
+    if (flush)
+        to.flush();
 
     while (true) {
 
@@ -803,9 +925,13 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
         }
 
         else if (msg == STDERR_ERROR) {
-            string error = readString(from);
-            unsigned int status = readInt(from);
-            return std::make_exception_ptr(Error(status, error));
+            if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
+                return std::make_exception_ptr(readError(from));
+            } else {
+                string error = readString(from);
+                unsigned int status = readInt(from);
+                return std::make_exception_ptr(Error(status, error));
+            }
         }
 
         else if (msg == STDERR_NEXT)
@@ -843,14 +969,49 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
     return nullptr;
 }
 
-static std::string uriScheme = "unix://";
-
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
+void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
 {
-    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
-    return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
-});
+    (*this)->to.flush();
+
+    std::exception_ptr ex;
+
+    /* Handle log messages / exceptions from the remote on a
+        separate thread. */
+    std::thread stderrThread([&]()
+    {
+        try {
+            processStderr(nullptr, nullptr, false);
+        } catch (...) {
+            ex = std::current_exception();
+        }
+    });
+
+    Finally joinStderrThread([&]()
+    {
+        if (stderrThread.joinable()) {
+            stderrThread.join();
+            if (ex) {
+                try {
+                    std::rethrow_exception(ex);
+                } catch (...) {
+                    ignoreException();
+                }
+            }
+        }
+    });
+
+    {
+        FramedSink sink((*this)->to, ex);
+        fun(sink);
+        sink.flush();
+    }
+
+    stderrThread.join();
+    if (ex)
+        std::rethrow_exception(ex);
+
+}
+
+static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
 
 }

src/libstore/remote-store.hh

@@ -16,19 +16,23 @@ struct FdSource;
 template<typename T> class Pool;
 struct ConnectionHandle;
 
+struct RemoteStoreConfig : virtual StoreConfig
+{
+    using StoreConfig::StoreConfig;
+
+    const Setting<int> maxConnections{(StoreConfig*) this, 1,
+            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
+
+    const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
+            "max-connection-age", "number of seconds to reuse a connection"};
+};
 
 /* FIXME: RemoteStore is a misnomer - should be something like
    DaemonStore. */
-class RemoteStore : public virtual Store
+class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
 {
 public:
 
-    const Setting<int> maxConnections{(Store*) this, 1,
-            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
-
-    const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
-            "max-connection-age", "number of seconds to reuse a connection"};
-
     virtual bool sameMachine() = 0;
 
     RemoteStore(const Params & params);
@@ -51,21 +55,28 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
-    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
+    std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
-    void querySubstitutablePathInfos(const StorePathSet & paths,
+    void querySubstitutablePathInfos(const StorePathCAMap & paths,
         SubstitutablePathInfos & infos) override;
 
-    void addToStore(const ValidPathInfo & info, Source & nar,
-        RepairFlag repair, CheckSigsFlag checkSigs,
-        std::shared_ptr<FSAccessor> accessor) override;
+    /* Add a content-addressable store path. `dump` will be drained. */
+    ref<const ValidPathInfo> addCAToStore(
+        Source & dump,
+        const string & name,
+        ContentAddressMethod caMethod,
+        const StorePathSet & references,
+        RepairFlag repair);
 
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
+    /* Add a content-addressable store path. Does not support references. `dump` will be drained. */
+    StorePath addToStoreFromDump(Source & dump, const string & name,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+
+    void addToStore(const ValidPathInfo & info, Source & nar,
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
@@ -95,7 +106,7 @@ public:
 
     void queryMissing(const std::vector<StorePathWithOutputs> & targets,
         StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
-        unsigned long long & downloadSize, unsigned long long & narSize) override;
+        uint64_t & downloadSize, uint64_t & narSize) override;
 
     void connect() override;
 
@@ -103,8 +114,6 @@ public:
 
     void flushBadConnections();
 
-protected:
-
     struct Connection
     {
         AutoCloseFD fd;
@@ -115,11 +124,13 @@ protected:
 
         virtual ~Connection();
 
-        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0);
+        std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
     };
 
     ref<Connection> openConnectionWrapper();
 
+protected:
+
     virtual ref<Connection> openConnection() = 0;
 
     void initConnection(Connection & conn);
@@ -132,24 +143,56 @@ protected:
 
     friend struct ConnectionHandle;
 
+    virtual ref<FSAccessor> getFSAccessor() override;
+
+    virtual void narFromPath(const StorePath & path, Sink & sink) override;
+
+    ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path);
+
 private:
 
     std::atomic_bool failed{false};
 
 };
 
-class UDSRemoteStore : public LocalFSStore, public RemoteStore
+struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
+{
+    UDSRemoteStoreConfig(const Store::Params & params)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , RemoteStoreConfig(params)
+    {
+    }
+
+    UDSRemoteStoreConfig()
+        : UDSRemoteStoreConfig(Store::Params({}))
+    {
+    }
+
+    const std::string name() override { return "Local Daemon Store"; }
+};
+
+class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
 {
 public:
 
     UDSRemoteStore(const Params & params);
-    UDSRemoteStore(std::string path, const Params & params);
+    UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
 
     std::string getUri() override;
 
+    static std::set<std::string> uriSchemes()
+    { return {"unix"}; }
+
     bool sameMachine() override
     { return true; }
 
+    ref<FSAccessor> getFSAccessor() override
+    { return LocalFSStore::getFSAccessor(); }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    { LocalFSStore::narFromPath(path, sink); }
+
 private:
 
     ref<RemoteStore::Connection> openConnection() override;

src/libstore/s3-binary-cache-store.cc

@@ -7,7 +7,6 @@
 #include "globals.hh"
 #include "compression.hh"
 #include "filetransfer.hh"
-#include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/VersionConfig.h>
@@ -173,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject(
     return res;
 }
 
-struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
+struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
-    const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
-    const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
-    const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
-    const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
-    const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
-    const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
-    const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
+    using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
+    const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
+    const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
+    const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
+    const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
+    const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
+    const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
+    const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
     const Setting<bool> multipartUpload{
-        this, false, "multipart-upload", "whether to use multi-part uploads"};
+        (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
     const Setting<uint64_t> bufferSize{
-        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
+        (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
 
+    const std::string name() override { return "S3 Binary Cache Store"; }
+};
+
+struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
+{
     std::string bucketName;
 
     Stats stats;
@@ -194,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     S3Helper s3Helper;
 
     S3BinaryCacheStoreImpl(
-        const Params & params, const std::string & bucketName)
-        : S3BinaryCacheStore(params)
+        const std::string & scheme,
+        const std::string & bucketName,
+        const Params & params)
+        : StoreConfig(params)
+        , S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(profile, region, scheme, endpoint)
     {
@@ -262,11 +270,14 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     std::shared_ptr<TransferManager> transferManager;
     std::once_flag transferManagerCreated;
 
-    void uploadFile(const std::string & path, const std::string & data,
+    void uploadFile(const std::string & path,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType,
         const std::string & contentEncoding)
     {
-        auto stream = std::make_shared<istringstream_nocopy>(data);
+        istream->seekg(0, istream->end);
+        auto size = istream->tellg();
+        istream->seekg(0, istream->beg);
 
         auto maxThreads = std::thread::hardware_concurrency();
 
@@ -307,7 +318,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
             std::shared_ptr<TransferHandle> transferHandle =
                 transferManager->UploadFile(
-                    stream, bucketName, path, mimeType,
+                    istream, bucketName, path, mimeType,
                     Aws::Map<Aws::String, Aws::String>(),
                     nullptr /*, contentEncoding */);
 
@@ -333,9 +344,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             if (contentEncoding != "")
                 request.SetContentEncoding(contentEncoding);
 
-            auto stream = std::make_shared<istringstream_nocopy>(data);
-
-            request.SetBody(stream);
+            request.SetBody(istream);
 
             auto result = checkAws(fmt("AWS error uploading '%s'", path),
                 s3Helper.client->PutObject(request));
@@ -347,25 +356,32 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
                 .count();
 
-        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
-                  bucketName % path % data.size() % duration);
+        printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
+            bucketName, path, size, duration);
 
         stats.putTimeMs += duration;
-        stats.putBytes += data.size();
+        stats.putBytes += std::max(size, (decltype(size)) 0);
         stats.put++;
     }
 
-    void upsertFile(const std::string & path, const std::string & data,
+    void upsertFile(const std::string & path,
+        std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType) override
     {
+        auto compress = [&](std::string compression)
+        {
+            auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
+            return std::make_shared<std::stringstream>(std::move(*compressed));
+        };
+
         if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
-            uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
+            uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
         else if (lsCompression != "" && hasSuffix(path, ".ls"))
-            uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
+            uploadFile(path, compress(lsCompression), mimeType, lsCompression);
         else if (logCompression != "" && hasPrefix(path, "log/"))
-            uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
+            uploadFile(path, compress(logCompression), mimeType, logCompression);
         else
-            uploadFile(path, data, mimeType, "");
+            uploadFile(path, istream, mimeType, "");
     }
 
     void getFile(const std::string & path, Sink & sink) override
@@ -410,7 +426,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             for (auto object : contents) {
                 auto & key = object.GetKey();
                 if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
-                paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
+                paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
             }
 
             marker = res.GetNextMarker();
@@ -419,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         return paths;
     }
 
+    static std::set<std::string> uriSchemes() { return {"s3"}; }
+
 };
 
-static RegisterStoreImplementation regStore([](
-    const std::string & uri, const Store::Params & params)
-    -> std::shared_ptr<Store>
-{
-    if (std::string(uri, 0, 5) != "s3://") return 0;
-    auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
-    store->init();
-    return store;
-});
+static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regS3BinaryCacheStore;
 
 }
 

src/libstore/sqlite.cc

@@ -61,6 +61,11 @@ void SQLite::exec(const std::string & stmt)
     });
 }
 
+uint64_t SQLite::getLastInsertedRowId()
+{
+    return sqlite3_last_insert_rowid(db);
+}
+
 void SQLiteStmt::create(sqlite3 * db, const string & sql)
 {
     checkInterrupt();
@@ -95,10 +100,10 @@ SQLiteStmt::Use::~Use()
     sqlite3_reset(stmt);
 }
 
-SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull)
+SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool notNull)
 {
     if (notNull) {
-        if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
+        if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
             throwSQLiteError(stmt.db, "binding argument");
     } else
         bind();

src/libstore/sqlite.hh

@@ -26,6 +26,8 @@ struct SQLite
     void isCache();
 
     void exec(const std::string & stmt);
+
+    uint64_t getLastInsertedRowId();
 };
 
 /* RAII wrapper to create and destroy SQLite prepared statements. */
@@ -54,7 +56,7 @@ struct SQLiteStmt
         ~Use();
 
         /* Bind the next parameter. */
-        Use & operator () (const std::string & value, bool notNull = true);
+        Use & operator () (std::string_view value, bool notNull = true);
         Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
         Use & operator () (int64_t value, bool notNull = true);
         Use & bind(); // null