Merge remote-tracking branch 'upstream/master' into overlayfs-store

src/libstore/binary-cache-store.cc

@@ -12,6 +12,7 @@
 #include "thread-pool.hh"
 #include "callback.hh"
 #include "signals.hh"
+#include "archive.hh"
 
 #include <chrono>
 #include <future>
@@ -27,7 +28,8 @@ BinaryCacheStore::BinaryCacheStore(const Params & params)
     , Store(params)
 {
     if (secretKeyFile != "")
-        secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
+        signer = std::make_unique<LocalSigner>(
+            SecretKey { readFile(secretKeyFile) });
 
     StringSink sink;
     sink << narVersionMagic1;
@@ -273,7 +275,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
-    if (secretKey) narInfo->sign(*this, *secretKey);
+    if (signer) narInfo->sign(*this, *signer);
 
     writeNarInfo(narInfo);
 
@@ -300,24 +302,60 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     }});
 }
 
-StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
-                                               FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
+StorePath BinaryCacheStore::addToStoreFromDump(
+    Source & dump,
+    std::string_view name,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
-    if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
-        unsupported("addToStoreFromDump");
-    return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
+    std::optional<Hash> caHash;
+    std::string nar;
+
+    if (auto * dump2p = dynamic_cast<StringSource *>(&dump)) {
+        auto & dump2 = *dump2p;
+        // Hack, this gives us a "replayable" source so we can compute
+        // multiple hashes more easily.
+        caHash = hashString(HashAlgorithm::SHA256, dump2.s);
+        switch (method.getFileIngestionMethod()) {
+        case FileIngestionMethod::Recursive:
+            // The dump is already NAR in this case, just use it.
+            nar = dump2.s;
+            break;
+        case FileIngestionMethod::Flat:
+            // The dump is Flat, so we need to convert it to NAR with a
+            // single file.
+            StringSink s;
+            dumpString(dump2.s, s);
+            nar = std::move(s.s);
+            break;
+        }
+    } else {
+        // Otherwise, we have to do th same hashing as NAR so our single
+        // hash will suffice for both purposes.
+        if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
+            unsupported("addToStoreFromDump");
+    }
+    StringSource narDump { nar };
+
+    // Use `narDump` if we wrote to `nar`.
+    Source & narDump2 = nar.size() > 0
+        ? static_cast<Source &>(narDump)
+        : dump;
+
+    return addToStoreCommon(narDump2, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
-            FixedOutputInfo {
-                .method = method,
-                .hash = nar.first,
-                .references = {
+            ContentAddressWithReferences::fromParts(
+                method,
+                caHash ? *caHash : nar.first,
+                {
                     .others = references,
                     // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
-                },
-            },
+                }),
             nar.first,
         };
         info.narSize = nar.second;
@@ -399,72 +437,36 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
-        HashAlgorithm hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references)
+    std::string_view name,
+    SourceAccessor & accessor,
+    const CanonPath & path,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    PathFilter & filter,
+    RepairFlag repair)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
        implementation of this method in terms of addToStoreFromDump. */
 
-    HashSink sink { hashAlgo };
-    if (method == FileIngestionMethod::Recursive) {
-        dumpPath(srcPath, sink, filter);
-    } else {
-        readFile(srcPath, sink);
-    }
-    auto h = sink.finish().first;
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
 
     auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(srcPath, sink, filter);
+        accessor.dumpPath(path, sink, filter);
     });
     return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
-            FixedOutputInfo {
-                .method = method,
-                .hash = h,
-                .references = {
+            ContentAddressWithReferences::fromParts(
+                method,
+                h,
+                {
                     .others = references,
                     // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
-                },
-            },
-            nar.first,
-        };
-        info.narSize = nar.second;
-        return info;
-    })->path;
-}
-
-StorePath BinaryCacheStore::addTextToStore(
-    std::string_view name,
-    std::string_view s,
-    const StorePathSet & references,
-    RepairFlag repair)
-{
-    auto textHash = hashString(HashAlgorithm::SHA256, s);
-    auto path = makeTextPath(name, TextInfo { { textHash }, references });
-
-    if (!repair && isValidPath(path))
-        return path;
-
-    StringSink sink;
-    dumpString(s, sink);
-    StringSource source(sink.s);
-    return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
-        ValidPathInfo info {
-            *this,
-            std::string { name },
-            TextInfo {
-                .hash = textHash,
-                .references = references,
-            },
+                }),
             nar.first,
         };
         info.narSize = nar.second;

src/libstore/binary-cache-store.hh

@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "crypto.hh"
+#include "signature/local-keys.hh"
 #include "store-api.hh"
 #include "log-store.hh"
 
@@ -57,8 +57,7 @@ class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
 {
 
 private:
-
-    std::unique_ptr<SecretKey> secretKey;
+    std::unique_ptr<Signer> signer;
 
 protected:
 
@@ -123,22 +122,22 @@ public:
     void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStoreFromDump(Source & dump, std::string_view name,
-                                 FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override;
+    StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        RepairFlag repair) override;
 
     StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            PathFilter & filter,
-            RepairFlag repair,
-            const StorePathSet & references) override;
-
-    StorePath addTextToStore(
         std::string_view name,
-        std::string_view s,
+        SourceAccessor & accessor,
+        const CanonPath & srcPath,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
         const StorePathSet & references,
+        PathFilter & filter,
         RepairFlag repair) override;
 
     void registerDrvOutput(const Realisation & info) override;

src/libstore/build/derivation-goal.cc

@@ -196,10 +196,19 @@ void DerivationGoal::loadDerivation()
        things being garbage collected while we're busy. */
     worker.evalStore.addTempRoot(drvPath);
 
-    assert(worker.evalStore.isValidPath(drvPath));
+    /* Get the derivation. It is probably in the eval store, but it might be inthe main store:
 
-    /* Get the derivation. */
-    drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath));
+         - Resolved derivation are resolved against main store realisations, and so must be stored there.
+
+         - Dynamic derivations are built, and so are found in the main store.
+     */
+    for (auto * drvStore : { &worker.evalStore, &worker.store }) {
+        if (drvStore->isValidPath(drvPath)) {
+            drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
+            break;
+        }
+    }
+    assert(drv);
 
     haveDerivation();
 }
@@ -214,7 +223,7 @@ void DerivationGoal::haveDerivation()
     if (!drv->type().hasKnownOutputPaths())
         experimentalFeatureSettings.require(Xp::CaDerivations);
 
-    if (!drv->type().isPure()) {
+    if (drv->type().isImpure()) {
         experimentalFeatureSettings.require(Xp::ImpureDerivations);
 
         for (auto & [outputName, output] : drv->outputs) {
@@ -295,7 +304,7 @@ void DerivationGoal::outputsSubstitutionTried()
 {
     trace("all outputs substituted (maybe)");
 
-    assert(drv->type().isPure());
+    assert(!drv->type().isImpure());
 
     if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
         done(BuildResult::TransientFailure, {},
@@ -388,9 +397,9 @@ void DerivationGoal::gaveUpOnSubstitution()
         for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs.map) {
             /* Ensure that pure, non-fixed-output derivations don't
                depend on impure derivations. */
-            if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
+            if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() && !drv->type().isFixed()) {
                 auto inputDrv = worker.evalStore.readDerivation(inputDrvPath);
-                if (!inputDrv.type().isPure())
+                if (inputDrv.type().isImpure())
                     throw Error("pure derivation '%s' depends on impure derivation '%s'",
                         worker.store.printStorePath(drvPath),
                         worker.store.printStorePath(inputDrvPath));
@@ -401,11 +410,15 @@ void DerivationGoal::gaveUpOnSubstitution()
     }
 
     /* Copy the input sources from the eval store to the build
-       store. */
+       store.
+
+       Note that some inputs might not be in the eval store because they
+       are (resolved) derivation outputs in a resolved derivation. */
     if (&worker.evalStore != &worker.store) {
         RealisedPath::Set inputSrcs;
         for (auto & i : drv->inputSrcs)
-            inputSrcs.insert(i);
+            if (worker.evalStore.isValidPath(i))
+                inputSrcs.insert(i);
         copyClosure(worker.evalStore, worker.store, inputSrcs);
     }
 
@@ -426,7 +439,7 @@ void DerivationGoal::gaveUpOnSubstitution()
 
 void DerivationGoal::repairClosure()
 {
-    assert(drv->type().isPure());
+    assert(!drv->type().isImpure());
 
     /* If we're repairing, we now know that our own outputs are valid.
        Now check whether the other paths in the outputs closure are
@@ -453,7 +466,7 @@ void DerivationGoal::repairClosure()
     std::map<StorePath, StorePath> outputsToDrv;
     for (auto & i : inputClosure)
         if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
+            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
             for (auto & j : depOutputs)
                 if (j.second)
                     outputsToDrv.insert_or_assign(*j.second, i);
@@ -604,7 +617,13 @@ void DerivationGoal::inputsRealised()
                     return *outPath;
                 }
                 else {
-                    auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
+                    auto outMap = [&]{
+                        for (auto * drvStore : { &worker.evalStore, &worker.store })
+                            if (drvStore->isValidPath(depDrvPath))
+                                return worker.store.queryDerivationOutputMap(depDrvPath, drvStore);
+                        assert(false);
+                    }();
+
                     auto outMapPath = outMap.find(outputName);
                     if (outMapPath == outMap.end()) {
                         throw Error(
@@ -1081,12 +1100,16 @@ void DerivationGoal::resolvedFinished()
                   worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName);
             }();
 
-            if (drv->type().isPure()) {
+            if (!drv->type().isImpure()) {
                 auto newRealisation = realisation;
                 newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
                 newRealisation.signatures.clear();
-                if (!drv->type().isFixed())
-                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
+                if (!drv->type().isFixed()) {
+                    auto & drvStore = worker.evalStore.isValidPath(drvPath)
+                        ? worker.evalStore
+                        : worker.store;
+                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
+                }
                 signRealisation(newRealisation);
                 worker.store.registerDrvOutput(newRealisation);
             }
@@ -1372,34 +1395,40 @@ void DerivationGoal::flushLine()
 
 std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
 {
-    assert(drv->type().isPure());
+    assert(!drv->type().isImpure());
     if (!useDerivation || drv->type().hasKnownOutputPaths()) {
         std::map<std::string, std::optional<StorePath>> res;
         for (auto & [name, output] : drv->outputs)
             res.insert_or_assign(name, output.path(worker.store, drv->name, name));
         return res;
     } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
+        for (auto * drvStore : { &worker.evalStore, &worker.store })
+            if (drvStore->isValidPath(drvPath))
+                return worker.store.queryPartialDerivationOutputMap(drvPath, drvStore);
+        assert(false);
     }
 }
 
 OutputPathMap DerivationGoal::queryDerivationOutputMap()
 {
-    assert(drv->type().isPure());
+    assert(!drv->type().isImpure());
     if (!useDerivation || drv->type().hasKnownOutputPaths()) {
         OutputPathMap res;
         for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
             res.insert_or_assign(name, *output.second);
         return res;
     } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
+        for (auto * drvStore : { &worker.evalStore, &worker.store })
+            if (drvStore->isValidPath(drvPath))
+                return worker.store.queryDerivationOutputMap(drvPath, drvStore);
+        assert(false);
     }
 }
 
 
 std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
 {
-    if (!drv->type().isPure()) return { false, {} };
+    if (drv->type().isImpure()) return { false, {} };
 
     bool checkHash = buildMode == bmRepair;
     auto wantedOutputsLeft = std::visit(overloaded {

src/libstore/build/entry-points.cc

@@ -26,9 +26,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         }
         if (i->exitCode != Goal::ecSuccess) {
             if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
-                failed.insert(std::string { i2->drvPath.to_string() });
+                failed.insert(printStorePath(i2->drvPath));
             else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
-                failed.insert(std::string { i2->storePath.to_string()});
+                failed.insert(printStorePath(i2->storePath));
         }
     }
 

src/libstore/build/local-derivation-goal.cc

@@ -20,6 +20,7 @@
 #include "child.hh"
 #include "unix-domain-socket.hh"
 #include "posix-fs-canonicalise.hh"
+#include "posix-source-accessor.hh"
 
 #include <regex>
 #include <queue>
@@ -1290,13 +1291,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
     { throw Error("queryPathFromHashPart"); }
 
     StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            PathFilter & filter,
-            RepairFlag repair,
-            const StorePathSet & references) override
+        std::string_view name,
+        SourceAccessor & accessor,
+        const CanonPath & srcPath,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        PathFilter & filter,
+        RepairFlag repair) override
     { throw Error("addToStore"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
@@ -1306,26 +1308,15 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
         goal.addDependency(info.path);
     }
 
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
     StorePath addToStoreFromDump(
-            Source & dump,
-            std::string_view name,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            RepairFlag repair,
-            const StorePathSet & references) override
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        RepairFlag repair) override
     {
-        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
+        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair);
         goal.addDependency(path);
         return path;
     }
@@ -1504,7 +1495,7 @@ void LocalDerivationGoal::startDaemon()
                     daemon::processConnection(store, from, to,
                         NotTrusted, daemon::Recursive);
                     debug("terminated daemon connection");
-                } catch (SysError &) {
+                } catch (SystemError &) {
                     ignoreException();
                 }
             });
@@ -1716,7 +1707,7 @@ void LocalDerivationGoal::runChild()
         try {
             if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
                 netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
 
 #if __linux__
         if (useChroot) {
@@ -2453,8 +2444,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 throw BuildError(
                     "output path %1% without valid stats info",
                     actualPath);
-            if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
-                outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
+            if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat)
             {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
@@ -2466,38 +2456,23 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             rewriteOutput(outputRewrites);
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
-            HashModuloSink caSink {outputHash.hashAlgo, oldHashPart };
-            std::visit(overloaded {
-                [&](const TextIngestionMethod &) {
-                    readFile(actualPath, caSink);
-                },
-                [&](const FileIngestionMethod & m2) {
-                    switch (m2) {
-                    case FileIngestionMethod::Recursive:
-                        dumpPath(actualPath, caSink);
-                        break;
-                    case FileIngestionMethod::Flat:
-                        readFile(actualPath, caSink);
-                        break;
-                    }
-                },
-            }, outputHash.method.raw);
-            auto got = caSink.finish().first;
+            auto got = ({
+                HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
+                PosixSourceAccessor accessor;
+                dumpPath(
+                    accessor, CanonPath { actualPath },
+                    caSink,
+                    outputHash.method.getFileIngestionMethod());
+                caSink.finish().first;
+            });
 
-            auto optCA = ContentAddressWithReferences::fromPartsOpt(
-                outputHash.method,
-                std::move(got),
-                rewriteRefs());
-            if (!optCA) {
-                // TODO track distinct failure modes separately (at the time of
-                // writing there is just one but `nullopt` is unclear) so this
-                // message can't get out of sync.
-                throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
-            }
             ValidPathInfo newInfo0 {
                 worker.store,
                 outputPathName(drv->name, outputName),
-                std::move(*optCA),
+                ContentAddressWithReferences::fromParts(
+                    outputHash.method,
+                    std::move(got),
+                    rewriteRefs()),
                 Hash::dummy,
             };
             if (*scratchPath != newInfo0.path) {
@@ -2511,9 +2486,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                                std::string(newInfo0.path.hashPart())}});
             }
 
-            HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath);
-            newInfo0.narHash = narHashAndSize.first;
-            newInfo0.narSize = narHashAndSize.second;
+            {
+                PosixSourceAccessor accessor;
+                HashResult narHashAndSize = hashPath(
+                    accessor, CanonPath { actualPath },
+                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
+                newInfo0.narHash = narHashAndSize.first;
+                newInfo0.narSize = narHashAndSize.second;
+            }
 
             assert(newInfo0.ca);
             return newInfo0;
@@ -2531,7 +2511,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                         std::string { scratchPath->hashPart() },
                         std::string { requiredFinalPath.hashPart() });
                 rewriteOutput(outputRewrites);
-                auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath);
+                PosixSourceAccessor accessor;
+                HashResult narHashAndSize = hashPath(
+                    accessor, CanonPath { actualPath },
+                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
                 ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
                 newInfo0.narSize = narHashAndSize.second;
                 auto refs = rewriteRefs();
@@ -2741,7 +2724,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             .outPath = newInfo.path
         };
         if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
-            && drv->type().isPure())
+            && !drv->type().isImpure())
         {
             signRealisation(thisRealisation);
             worker.store.registerDrvOutput(thisRealisation);

src/libstore/build/sandbox-defaults.sb

@@ -68,6 +68,7 @@ R""(
 (allow file*
        (literal "/dev/null")
        (literal "/dev/random")
+       (literal "/dev/stderr")
        (literal "/dev/stdin")
        (literal "/dev/stdout")
        (literal "/dev/tty")

src/libstore/build/substitution-goal.cc

@@ -2,6 +2,7 @@
 #include "substitution-goal.hh"
 #include "nar-info.hh"
 #include "finally.hh"
+#include "signals.hh"
 
 namespace nix {
 
@@ -217,6 +218,8 @@ void PathSubstitutionGoal::tryToRun()
 
     thr = std::thread([this]() {
         try {
+            ReceiveInterrupts receiveInterrupts;
+
             /* Wake up the worker loop when we're done. */
             Finally updateStats([this]() { outPipe.writeSide.close(); });
 

src/libstore/build/worker.cc

@@ -251,7 +251,7 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
 
 void Worker::waitForBuildSlot(GoalPtr goal)
 {
-    debug("wait for build slot");
+    goal->trace("wait for build slot");
     bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution;
     if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) ||
         (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))
@@ -449,7 +449,7 @@ void Worker::waitForInput()
                 } else {
                     printMsg(lvlVomit, "%1%: read %2% bytes",
                         goal->getName(), rd);
-                    std::string data((char *) buffer.data(), rd);
+                    std::string_view data((char *) buffer.data(), rd);
                     j->lastOutput = after;
                     goal->handleChildOutput(k, data);
                 }
@@ -519,7 +519,9 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(info->narHash.algo, store.printStorePath(path));
+        HashResult current = hashPath(
+            *store.getFSAccessor(), CanonPath { store.printStorePath(path) },
+            FileIngestionMethod::Recursive, info->narHash.algo);
         Hash nullHash(HashAlgorithm::SHA256);
         res = info->narHash == nullHash || info->narHash == current.first;
     }

src/libstore/build/worker.hh

@@ -116,7 +116,7 @@ private:
     WeakGoals waitingForAWhile;
 
     /**
-     * Last time the goals in `waitingForAWhile` where woken up.
+     * Last time the goals in `waitingForAWhile` were woken up.
      */
     steady_time_point lastWokenUp;
 

src/libstore/builtins/buildenv.hh

@@ -5,6 +5,9 @@
 
 namespace nix {
 
+/**
+ * Think of this as a "store level package attrset", but stripped down to no more than the needs of buildenv.
+ */
 struct Package {
     Path path;
     bool active;

src/libstore/content-address.cc

@@ -50,6 +50,18 @@ std::string ContentAddressMethod::render(HashAlgorithm ha) const
     }, raw);
 }
 
+FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const
+{
+    return std::visit(overloaded {
+        [&](const TextIngestionMethod & th) {
+            return FileIngestionMethod::Flat;
+        },
+        [&](const FileIngestionMethod & fim) {
+            return fim;
+        }
+    }, raw);
+}
+
 std::string ContentAddress::render() const
 {
     return std::visit(overloaded {
@@ -79,7 +91,7 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
         prefix = *optPrefix;
     }
 
-    auto parseHashType_ = [&](){
+    auto parseHashAlgorithm_ = [&](){
         auto hashTypeRaw = splitPrefixTo(rest, ':');
         if (!hashTypeRaw)
             throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
@@ -90,7 +102,7 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
     // Switch on prefix
     if (prefix == "text") {
         // No parsing of the ingestion method, "text" only support flat.
-        HashAlgorithm hashAlgo = parseHashType_();
+        HashAlgorithm hashAlgo = parseHashAlgorithm_();
         return {
             TextIngestionMethod {},
             std::move(hashAlgo),
@@ -100,7 +112,7 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
         auto method = FileIngestionMethod::Flat;
         if (splitPrefix(rest, "r:"))
             method = FileIngestionMethod::Recursive;
-        HashAlgorithm hashAlgo = parseHashType_();
+        HashAlgorithm hashAlgo = parseHashAlgorithm_();
         return {
             std::move(method),
             std::move(hashAlgo),
@@ -176,13 +188,13 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con
     }, ca.method.raw);
 }
 
-std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
-    ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept
+ContentAddressWithReferences ContentAddressWithReferences::fromParts(
+    ContentAddressMethod method, Hash hash, StoreReferences refs)
 {
     return std::visit(overloaded {
-        [&](TextIngestionMethod _) -> std::optional<ContentAddressWithReferences> {
+        [&](TextIngestionMethod _) -> ContentAddressWithReferences {
             if (refs.self)
-                return std::nullopt;
+                throw Error("self-reference not allowed with text hashing");
             return ContentAddressWithReferences {
                 TextInfo {
                     .hash = std::move(hash),
@@ -190,7 +202,7 @@ std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPa
                 }
             };
         },
-        [&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
+        [&](FileIngestionMethod m2) -> ContentAddressWithReferences {
             return ContentAddressWithReferences {
                 FixedOutputInfo {
                     .method = m2,

src/libstore/content-address.hh

@@ -4,6 +4,7 @@
 #include <variant>
 #include "hash.hh"
 #include "path.hh"
+#include "file-content-address.hh"
 #include "comparator.hh"
 #include "variant-wrapper.hh"
 
@@ -31,22 +32,6 @@ namespace nix {
  */
 struct TextIngestionMethod : std::monostate { };
 
-/**
- * An enumeration of the main ways we can serialize file system
- * objects.
- */
-enum struct FileIngestionMethod : uint8_t {
-    /**
-     * Flat-file hashing. Directly ingest the contents of a single file
-     */
-    Flat = 0,
-    /**
-     * Recursive (or NAR) hashing. Serializes the file-system object in Nix
-     * Archive format and ingest that
-     */
-    Recursive = 1
-};
-
 /**
  * Compute the prefix to the hash algorithm which indicates how the
  * files were ingested.
@@ -54,7 +39,7 @@ enum struct FileIngestionMethod : uint8_t {
 std::string makeFileIngestionPrefix(FileIngestionMethod m);
 
 /**
- * An enumeration of all the ways we can serialize file system objects.
+ * An enumeration of all the ways we can content-address store objects.
  *
  * Just the type of a content address. Combine with the hash itself, and
  * we have a `ContentAddress` as defined below. Combine that, in turn,
@@ -102,7 +87,15 @@ struct ContentAddressMethod
      *
      * The rough inverse of `parse()`.
      */
-    std::string render(HashAlgorithm ha) const;
+    std::string render(HashAlgorithm ht) const;
+
+    /**
+     * Get the underlying way to content-address file system objects.
+     *
+     * Different ways of hashing store objects may use the same method
+     * for hashing file systeme objects.
+     */
+    FileIngestionMethod getFileIngestionMethod() const;
 };
 
 
@@ -116,11 +109,11 @@ struct ContentAddressMethod
  * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the
  * following forms:
  *
- * - ‘text:sha256:<sha256 hash of file contents>’: For paths
- *   computed by Store::makeTextPath() / Store::addTextToStore().
+ * - `TextIngestionMethod`:
+ *   ‘text:sha256:<sha256 hash of file contents>’
  *
- * - ‘fixed:<r?>:<ht>:<h>’: For paths computed by
- *   Store::makeFixedOutputPath() / Store::addToStore().
+ * - `FixedIngestionMethod`:
+ *   ‘fixed:<r?>:<hash type>:<hash of file contents>’
  */
 struct ContentAddress
 {
@@ -266,11 +259,12 @@ struct ContentAddressWithReferences
      *
      * @param refs References to other store objects or oneself.
      *
-     * Do note that not all combinations are supported; `nullopt` is
-     * returns for invalid combinations.
+     * @note note that all combinations are supported. This is a
+     * *partial function* and exceptions will be thrown for invalid
+     * combinations.
      */
-    static std::optional<ContentAddressWithReferences> fromPartsOpt(
-        ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept;
+    static ContentAddressWithReferences fromParts(
+        ContentAddressMethod method, Hash hash, StoreReferences refs);
 
     ContentAddressMethod getMethod() const;
 

src/libstore/crypto.cc

@@ -1,117 +0,0 @@
-#include "crypto.hh"
-#include "file-system.hh"
-#include "util.hh"
-#include "globals.hh"
-
-#include <sodium.h>
-
-namespace nix {
-
-static std::pair<std::string_view, std::string_view> split(std::string_view s)
-{
-    size_t colon = s.find(':');
-    if (colon == std::string::npos || colon == 0)
-        return {"", ""};
-    return {s.substr(0, colon), s.substr(colon + 1)};
-}
-
-Key::Key(std::string_view s)
-{
-    auto ss = split(s);
-
-    name = ss.first;
-    key = ss.second;
-
-    if (name == "" || key == "")
-        throw Error("secret key is corrupt");
-
-    key = base64Decode(key);
-}
-
-std::string Key::to_string() const
-{
-    return name + ":" + base64Encode(key);
-}
-
-SecretKey::SecretKey(std::string_view s)
-    : Key(s)
-{
-    if (key.size() != crypto_sign_SECRETKEYBYTES)
-        throw Error("secret key is not valid");
-}
-
-std::string SecretKey::signDetached(std::string_view data) const
-{
-    unsigned char sig[crypto_sign_BYTES];
-    unsigned long long sigLen;
-    crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(),
-        (unsigned char *) key.data());
-    return name + ":" + base64Encode(std::string((char *) sig, sigLen));
-}
-
-PublicKey SecretKey::toPublicKey() const
-{
-    unsigned char pk[crypto_sign_PUBLICKEYBYTES];
-    crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data());
-    return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES));
-}
-
-SecretKey SecretKey::generate(std::string_view name)
-{
-    unsigned char pk[crypto_sign_PUBLICKEYBYTES];
-    unsigned char sk[crypto_sign_SECRETKEYBYTES];
-    if (crypto_sign_keypair(pk, sk) != 0)
-        throw Error("key generation failed");
-
-    return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES));
-}
-
-PublicKey::PublicKey(std::string_view s)
-    : Key(s)
-{
-    if (key.size() != crypto_sign_PUBLICKEYBYTES)
-        throw Error("public key is not valid");
-}
-
-bool verifyDetached(const std::string & data, const std::string & sig,
-    const PublicKeys & publicKeys)
-{
-    auto ss = split(sig);
-
-    auto key = publicKeys.find(std::string(ss.first));
-    if (key == publicKeys.end()) return false;
-
-    auto sig2 = base64Decode(ss.second);
-    if (sig2.size() != crypto_sign_BYTES)
-        throw Error("signature is not valid");
-
-    return crypto_sign_verify_detached((unsigned char *) sig2.data(),
-        (unsigned char *) data.data(), data.size(),
-        (unsigned char *) key->second.key.data()) == 0;
-}
-
-PublicKeys getDefaultPublicKeys()
-{
-    PublicKeys publicKeys;
-
-    // FIXME: filter duplicates
-
-    for (auto s : settings.trustedPublicKeys.get()) {
-        PublicKey key(s);
-        publicKeys.emplace(key.name, key);
-    }
-
-    for (auto secretKeyFile : settings.secretKeyFiles.get()) {
-        try {
-            SecretKey secretKey(readFile(secretKeyFile));
-            publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
-        } catch (SysError & e) {
-            /* Ignore unreadable key files. That's normal in a
-               multi-user installation. */
-        }
-    }
-
-    return publicKeys;
-}
-
-}

src/libstore/crypto.hh

@@ -1,69 +0,0 @@
-#pragma once
-///@file
-
-#include "types.hh"
-
-#include <map>
-
-namespace nix {
-
-struct Key
-{
-    std::string name;
-    std::string key;
-
-    /**
-     * Construct Key from a string in the format
-     * ‘<name>:<key-in-base64>’.
-     */
-    Key(std::string_view s);
-
-    std::string to_string() const;
-
-protected:
-    Key(std::string_view name, std::string && key)
-        : name(name), key(std::move(key)) { }
-};
-
-struct PublicKey;
-
-struct SecretKey : Key
-{
-    SecretKey(std::string_view s);
-
-    /**
-     * Return a detached signature of the given string.
-     */
-    std::string signDetached(std::string_view s) const;
-
-    PublicKey toPublicKey() const;
-
-    static SecretKey generate(std::string_view name);
-
-private:
-    SecretKey(std::string_view name, std::string && key)
-        : Key(name, std::move(key)) { }
-};
-
-struct PublicKey : Key
-{
-    PublicKey(std::string_view data);
-
-private:
-    PublicKey(std::string_view name, std::string && key)
-        : Key(name, std::move(key)) { }
-    friend struct SecretKey;
-};
-
-typedef std::map<std::string, PublicKey> PublicKeys;
-
-/**
- * @return true iff ‘sig’ is a correct signature over ‘data’ using one
- * of the given public keys.
- */
-bool verifyDetached(const std::string & data, const std::string & sig,
-    const PublicKeys & publicKeys);
-
-PublicKeys getDefaultPublicKeys();
-
-}

src/libstore/daemon.cc

@@ -403,22 +403,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr);
                 auto hashAlgo = hashAlgo_; // work around clang bug
                 FramedSource source(from);
-                // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
-                return std::visit(overloaded {
-                    [&](const TextIngestionMethod &) {
-                        if (hashAlgo != HashAlgorithm::SHA256)
-                            throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
-                                name, printHashAlgo(hashAlgo));
-                        // We could stream this by changing Store
-                        std::string contents = source.drain();
-                        auto path = store->addTextToStore(name, contents, refs, repair);
-                        return store->queryPathInfo(path);
-                    },
-                    [&](const FileIngestionMethod & fim) {
-                        auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs);
-                        return store->queryPathInfo(path);
-                    },
-                }, contentAddressMethod.raw);
+                // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
+                auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair);
+                return store->queryPathInfo(path);
             }();
             logger->stopWork();
 
@@ -454,7 +441,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                        eagerly consume the entire stream it's given, past the
                        length of the Nar. */
                     TeeSource savedNARSource(from, saved);
-                    NullParseSink sink; /* just parse the NAR */
+                    NullFileSystemObjectSink sink; /* just parse the NAR */
                     parseDump(sink, savedNARSource);
                 } else {
                     /* Incrementally parse the NAR file, stripping the
@@ -496,7 +483,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         std::string s = readString(from);
         auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
         logger->startWork();
-        auto path = store->addTextToStore(suffix, s, refs, NoRepair);
+        auto path = ({
+            StringSource source { s };
+            store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
+        });
         logger->stopWork();
         to << store->printStorePath(path);
         break;
@@ -923,7 +913,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 source = std::make_unique<TunnelSource>(from, to);
             else {
                 TeeSource tee { from, saved };
-                NullParseSink ether;
+                NullFileSystemObjectSink ether;
                 parseDump(ether, tee);
                 source = std::make_unique<StringSource>(saved.s);
             }

src/libstore/derivations.cc

@@ -2,6 +2,7 @@
 #include "downstream-placeholder.hh"
 #include "store-api.hh"
 #include "globals.hh"
+#include "types.hh"
 #include "util.hh"
 #include "split.hh"
 #include "common-protocol.hh"
@@ -109,17 +110,17 @@ bool DerivationType::isSandboxed() const
 }
 
 
-bool DerivationType::isPure() const
+bool DerivationType::isImpure() const
 {
     return std::visit(overloaded {
         [](const InputAddressed & ia) {
-            return true;
+            return false;
         },
         [](const ContentAddressed & ca) {
-            return true;
+            return false;
         },
         [](const Impure &) {
-            return false;
+            return true;
         },
     }, raw);
 }
@@ -143,36 +144,89 @@ StorePath writeDerivation(Store & store,
     auto suffix = std::string(drv.name) + drvExtension;
     auto contents = drv.unparse(store, false);
     return readOnly || settings.readOnlyMode
-        ? store.computeStorePathForText(suffix, contents, references)
-        : store.addTextToStore(suffix, contents, references, repair);
+        ? store.makeFixedOutputPathFromCA(suffix, TextInfo {
+            .hash = hashString(HashAlgorithm::SHA256, contents),
+            .references = std::move(references),
+        })
+        : ({
+            StringSource s { contents };
+            store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
+        });
+}
+
+
+namespace {
+/**
+ * This mimics std::istream to some extent. We use this much smaller implementation
+ * instead of plain istreams because the sentry object overhead is too high.
+ */
+struct StringViewStream {
+    std::string_view remaining;
+
+    int peek() const {
+        return remaining.empty() ? EOF : remaining[0];
+    }
+
+    int get() {
+        if (remaining.empty()) return EOF;
+        char c = remaining[0];
+        remaining.remove_prefix(1);
+        return c;
+    }
+};
+
+constexpr struct Escapes {
+    char map[256];
+    constexpr Escapes() {
+        for (int i = 0; i < 256; i++) map[i] = (char) (unsigned char) i;
+        map[(int) (unsigned char) 'n'] = '\n';
+        map[(int) (unsigned char) 'r'] = '\r';
+        map[(int) (unsigned char) 't'] = '\t';
+    }
+    char operator[](char c) const { return map[(unsigned char) c]; }
+} escapes;
 }
 
 
 /* Read string `s' from stream `str'. */
-static void expect(std::istream & str, std::string_view s)
+static void expect(StringViewStream & str, std::string_view s)
 {
-    for (auto & c : s) {
-        if (str.get() != c)
-            throw FormatError("expected string '%1%'", s);
-    }
+    if (!str.remaining.starts_with(s))
+        throw FormatError("expected string '%1%'", s);
+    str.remaining.remove_prefix(s.size());
 }
 
 
 /* Read a C-style string from stream `str'. */
-static std::string parseString(std::istream & str)
+static BackedStringView parseString(StringViewStream & str)
 {
-    std::string res;
     expect(str, "\"");
-    int c;
-    while ((c = str.get()) != '"')
-        if (c == '\\') {
-            c = str.get();
-            if (c == 'n') res += '\n';
-            else if (c == 'r') res += '\r';
-            else if (c == 't') res += '\t';
-            else res += c;
+    auto c = str.remaining.begin(), end = str.remaining.end();
+    bool escaped = false;
+    for (; c != end && *c != '"'; c++) {
+        if (*c == '\\') {
+            c++;
+            if (c == end)
+                throw FormatError("unterminated string in derivation");
+            escaped = true;
         }
-        else res += c;
+    }
+
+    const auto contentLen = c - str.remaining.begin();
+    const auto content = str.remaining.substr(0, contentLen);
+    str.remaining.remove_prefix(contentLen + 1);
+
+    if (!escaped)
+        return content;
+
+    std::string res;
+    res.reserve(content.size());
+    for (c = content.begin(), end = content.end(); c != end; c++)
+        if (*c == '\\') {
+            c++;
+            res += escapes[*c];
+        }
+        else res += *c;
     return res;
 }
 
@@ -181,15 +235,15 @@ static void validatePath(std::string_view s) {
         throw FormatError("bad path '%1%' in derivation", s);
 }
 
-static Path parsePath(std::istream & str)
+static BackedStringView parsePath(StringViewStream & str)
 {
     auto s = parseString(str);
-    validatePath(s);
+    validatePath(*s);
     return s;
 }
 
 
-static bool endOfList(std::istream & str)
+static bool endOfList(StringViewStream & str)
 {
     if (str.peek() == ',') {
         str.get();
@@ -203,12 +257,12 @@ static bool endOfList(std::istream & str)
 }
 
 
-static StringSet parseStrings(std::istream & str, bool arePaths)
+static StringSet parseStrings(StringViewStream & str, bool arePaths)
 {
     StringSet res;
     expect(str, "[");
     while (!endOfList(str))
-        res.insert(arePaths ? parsePath(str) : parseString(str));
+        res.insert((arePaths ? parsePath(str) : parseString(str)).toOwned());
     return res;
 }
 
@@ -261,7 +315,7 @@ static DerivationOutput parseDerivationOutput(
 }
 
 static DerivationOutput parseDerivationOutput(
-    const StoreDirConfig & store, std::istringstream & str,
+    const StoreDirConfig & store, StringViewStream & str,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
 {
     expect(str, ","); const auto pathS = parseString(str);
@@ -269,7 +323,7 @@ static DerivationOutput parseDerivationOutput(
     expect(str, ","); const auto hash = parseString(str);
     expect(str, ")");
 
-    return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings);
+    return parseDerivationOutput(store, *pathS, *hashAlgo, *hash, xpSettings);
 }
 
 /**
@@ -291,7 +345,7 @@ enum struct DerivationATermVersion {
 
 static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
     const StoreDirConfig & store,
-    std::istringstream & str,
+    StringViewStream & str,
     DerivationATermVersion version)
 {
     DerivedPathMap<StringSet>::ChildNode node;
@@ -317,7 +371,7 @@ static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
             expect(str, ",[");
             while (!endOfList(str)) {
                 expect(str, "(");
-                auto outputName = parseString(str);
+                auto outputName = parseString(str).toOwned();
                 expect(str, ",");
                 node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version));
                 expect(str, ")");
@@ -343,7 +397,7 @@ Derivation parseDerivation(
     Derivation drv;
     drv.name = name;
 
-    std::istringstream str(std::move(s));
+    StringViewStream str{s};
     expect(str, "D");
     DerivationATermVersion version;
     switch (str.peek()) {
@@ -354,12 +408,12 @@ Derivation parseDerivation(
     case 'r': {
         expect(str, "rvWithVersion(");
         auto versionS = parseString(str);
-        if (versionS == "xp-dyn-drv") {
+        if (*versionS == "xp-dyn-drv") {
             // Only verison we have so far
             version = DerivationATermVersion::DynamicDerivations;
             xpSettings.require(Xp::DynamicDerivations);
         } else {
-            throw FormatError("Unknown derivation ATerm format version '%s'", versionS);
+            throw FormatError("Unknown derivation ATerm format version '%s'", *versionS);
         }
         expect(str, ",");
         break;
@@ -371,7 +425,7 @@ Derivation parseDerivation(
     /* Parse the list of outputs. */
     expect(str, "[");
     while (!endOfList(str)) {
-        expect(str, "("); std::string id = parseString(str);
+        expect(str, "("); std::string id = parseString(str).toOwned();
         auto output = parseDerivationOutput(store, str, xpSettings);
         drv.outputs.emplace(std::move(id), std::move(output));
     }
@@ -380,28 +434,28 @@ Derivation parseDerivation(
     expect(str, ",[");
     while (!endOfList(str)) {
         expect(str, "(");
-        Path drvPath = parsePath(str);
+        auto drvPath = parsePath(str);
         expect(str, ",");
-        drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version));
+        drv.inputDrvs.map.insert_or_assign(store.parseStorePath(*drvPath), parseDerivedPathMapNode(store, str, version));
         expect(str, ")");
     }
 
     expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
-    expect(str, ","); drv.platform = parseString(str);
-    expect(str, ","); drv.builder = parseString(str);
+    expect(str, ","); drv.platform = parseString(str).toOwned();
+    expect(str, ","); drv.builder = parseString(str).toOwned();
 
     /* Parse the builder arguments. */
     expect(str, ",[");
     while (!endOfList(str))
-        drv.args.push_back(parseString(str));
+        drv.args.push_back(parseString(str).toOwned());
 
     /* Parse the environment variables. */
     expect(str, ",[");
     while (!endOfList(str)) {
-        expect(str, "("); auto name = parseString(str);
-        expect(str, ","); auto value = parseString(str);
+        expect(str, "("); auto name = parseString(str).toOwned();
+        expect(str, ","); auto value = parseString(str).toOwned();
         expect(str, ")");
-        drv.env[name] = value;
+        drv.env.insert_or_assign(std::move(name), std::move(value));
     }
 
     expect(str, ")");
@@ -786,7 +840,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         };
     }
 
-    if (!type.isPure()) {
+    if (type.isImpure()) {
         std::map<std::string, Hash> outputHashes;
         for (const auto & [outputName, _] : drv.outputs)
             outputHashes.insert_or_assign(outputName, impureOutputHash);

src/libstore/derivations.hh

@@ -253,12 +253,17 @@ struct DerivationType {
     bool isSandboxed() const;
 
     /**
-     * Whether the derivation is expected to produce the same result
-     * every time, and therefore it only needs to be built once. This is
-     * only false for derivations that have the attribute '__impure =
+     * Whether the derivation is expected to produce a different result
+     * every time, and therefore it needs to be rebuilt every time. This is
+     * only true for derivations that have the attribute '__impure =
      * true'.
+     *
+     * Non-impure derivations can still behave impurely, to the degree permitted
+     * by the sandbox. Hence why this method isn't `isPure`: impure derivations
+     * are not the negation of pure derivations. Purity can not be ascertained
+     * except by rather heavy tools.
      */
-    bool isPure() const;
+    bool isImpure() const;
 
     /**
      * Does the derivation knows its own output paths?

src/libstore/derived-path.cc

@@ -12,9 +12,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
@@ -22,13 +22,9 @@ namespace nix {
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
 
-#define FIELD_TYPE std::string
 CMP(SingleDerivedPath, SingleDerivedPathBuilt, output)
-#undef FIELD_TYPE
 
-#define FIELD_TYPE OutputsSpec
 CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
-#undef FIELD_TYPE
 
 #undef CMP
 #undef CMP_ONE

src/libstore/dummy-store.cc

@@ -58,12 +58,14 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
         RepairFlag repair, CheckSigsFlag checkSigs) override
     { unsupported("addToStore"); }
 
-    StorePath addTextToStore(
+    virtual StorePath addToStoreFromDump(
+        Source & dump,
         std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override
-    { unsupported("addTextToStore"); }
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        RepairFlag repair = NoRepair) override
+    { unsupported("addToStore"); }
 
     void narFromPath(const StorePath & path, Sink & sink) override
     { unsupported("narFromPath"); }

src/libstore/export-import.cc

@@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
         /* Extract the NAR from the source. */
         StringSink saved;
         TeeSource tee { source, saved };
-        NullParseSink ether;
+        NullFileSystemObjectSink ether;
         parseDump(ether, tee);
 
         uint32_t magic = readInt(source);

src/libstore/gc.cc

@@ -149,11 +149,12 @@ void LocalStore::addTempRoot(const StorePath & path)
             try {
                 nix::connect(fdRootsSocket->get(), socketPath);
             } catch (SysError & e) {
-                /* The garbage collector may have exited, so we need to
-                   restart. */
-                if (e.errNo == ECONNREFUSED) {
-                    debug("GC socket connection refused");
+                /* The garbage collector may have exited or not
+                   created the socket yet, so we need to restart. */
+                if (e.errNo == ECONNREFUSED || e.errNo == ENOENT) {
+                    debug("GC socket connection refused: %s", e.msg());
                     fdRootsSocket->close();
+                    std::this_thread::sleep_for(std::chrono::milliseconds(100));
                     goto restart;
                 }
                 throw;
@@ -413,7 +414,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
                         unchecked[i->str()].emplace(envFile);
-                } catch (SysError & e) {
+                } catch (SystemError & e) {
                     if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
                     throw;
@@ -509,6 +510,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     auto fdGCLock = openGCLock();
     FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock...");
 
+    /* Synchronisation point to test ENOENT handling in
+       addTempRoot(), see tests/gc-non-blocking.sh. */
+    if (auto p = getEnv("_NIX_TEST_GC_SYNC_1"))
+        readFile(*p);
+
     /* Start the server for receiving new roots. */
     auto socketPath = stateDir.get() + gcSocketPath;
     createDirs(dirOf(socketPath));
@@ -632,6 +638,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         roots.insert(root.first);
     }
 
+    /* Synchronisation point for testing, see tests/functional/gc-non-blocking.sh. */
+    if (auto p = getEnv("_NIX_TEST_GC_SYNC_2"))
+        readFile(*p);
+
     /* Helper function that deletes a path from the store and throws
        GCLimitReached if we've deleted enough garbage. */
     auto deleteFromStore = [&](std::string_view baseName)
@@ -779,10 +789,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         }
     };
 
-    /* Synchronisation point for testing, see tests/functional/gc-concurrent.sh. */
-    if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
-        readFile(*p);
-
     /* Either delete all garbage paths, or just the specified
        paths (for gcDeleteSpecific). */
     if (options.action == GCOptions::gcDeleteSpecific) {

src/libstore/globals.cc

@@ -15,8 +15,6 @@
 
 #include <nlohmann/json.hpp>
 
-#include <sodium/core.h>
-
 #ifdef __GLIBC__
 # include <gnu/lib-names.h>
 # include <nss.h>
@@ -120,7 +118,7 @@ void loadConfFile()
         try {
             std::string contents = readFile(path);
             globalConfig.applyConfig(contents, path);
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
     };
 
     applyConfigFile(settings.nixConfDir + "/nix.conf");
@@ -409,9 +407,6 @@ void initLibStore() {
 
     initLibUtil();
 
-    if (sodium_init() == -1)
-        throw Error("could not initialise libsodium");
-
     loadConfFile();
 
     preloadNSS();

src/libstore/globals.hh

@@ -144,20 +144,25 @@ public:
      */
     bool verboseBuild = true;
 
-    Setting<size_t> logLines{this, 10, "log-lines",
+    Setting<size_t> logLines{this, 25, "log-lines",
         "The number of lines of the tail of "
         "the log to show if a build fails."};
 
     MaxBuildJobsSetting maxBuildJobs{
         this, 1, "max-jobs",
         R"(
-          This option defines the maximum number of jobs that Nix will try to
-          build in parallel. The default is `1`. The special value `auto`
-          causes Nix to use the number of CPUs in your system. `0` is useful
-          when using remote builders to prevent any local builds (except for
-          `preferLocalBuild` derivation attribute which executes locally
-          regardless). It can be overridden using the `--max-jobs` (`-j`)
-          command line switch.
+          Maximum number of jobs that Nix will try to build locally in parallel.
+
+          The special value `auto` causes Nix to use the number of CPUs in your system.
+          Use `0` to disable local builds and directly use the remote machines specified in [`builders`](#conf-builders).
+          This will not affect derivations that have [`preferLocalBuild = true`](@docroot@/language/advanced-attributes.md#adv-attr-preferLocalBuild), which are always built locally.
+
+          > **Note**
+          >
+          > The number of CPU cores to use for each build job is independently determined by the [`cores`](#conf-cores) setting.
+
+          <!-- TODO(@fricklerhandwerk): would be good to have those shorthands for common options as part of the specification -->
+          The setting can be overridden using the `--max-jobs` (`-j`) command line switch.
         )",
         {"build-max-jobs"}};
 
@@ -214,7 +219,11 @@ public:
           In general, you do not have to modify this setting.
           While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong.
 
-          This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem).
+          This value is available in the Nix language as
+          [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+          if the
+          [`eval-system`](#conf-eval-system)
+          configuration option is set as the empty string.
         )"};
 
     Setting<time_t> maxSilentTime{
@@ -268,21 +277,16 @@ public:
     Setting<bool> alwaysAllowSubstitutes{
         this, false, "always-allow-substitutes",
         R"(
-          If set to `true`, Nix will ignore the `allowSubstitutes` attribute in
-          derivations and always attempt to use available substituters.
-          For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md).
+          If set to `true`, Nix will ignore the [`allowSubstitutes`](@docroot@/language/advanced-attributes.md) attribute in derivations and always attempt to use [available substituters](#conf-substituters).
         )"};
 
     Setting<bool> buildersUseSubstitutes{
         this, false, "builders-use-substitutes",
         R"(
-          If set to `true`, Nix will instruct remote build machines to use
-          their own binary substitutes if available. In practical terms, this
-          means that remote hosts will fetch as many build dependencies as
-          possible from their own substitutes (e.g, from `cache.nixos.org`),
-          instead of waiting for this host to upload them all. This can
-          drastically reduce build times if the network connection between
-          this computer and the remote build host is slow.
+          If set to `true`, Nix will instruct [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available.
+
+          It means that remote build hosts will fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all.
+          This can drastically reduce build times if the network connection between the local machine and the remote build host is slow.
         )"};
 
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
@@ -632,11 +636,11 @@ public:
 
           At least one of the following condition must be met
           for Nix to accept copying a store object from another
-          Nix store (such as a substituter):
+          Nix store (such as a [substituter](#conf-substituters)):
 
           - the store object has been signed using a key in the trusted keys list
           - the [`require-sigs`](#conf-require-sigs) option has been set to `false`
-          - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object)
+          - the store object is [content-addressed](@docroot@/glossary.md#gloss-content-addressed-store-object)
         )",
         {"binary-cache-public-keys"}};
 
@@ -947,7 +951,9 @@ public:
           may be useful in certain scenarios (e.g. to spin up containers or
           set up userspace network interfaces in tests).
         )"};
+#endif
 
+#if HAVE_ACL_SUPPORT
     Setting<StringSet> ignoredAcls{
         this, {"security.selinux", "system.nfs4_acl", "security.csm"}, "ignored-acls",
         R"(

src/libstore/keys.cc

@@ -0,0 +1,31 @@
+#include "file-system.hh"
+#include "globals.hh"
+#include "keys.hh"
+
+namespace nix {
+
+PublicKeys getDefaultPublicKeys()
+{
+    PublicKeys publicKeys;
+
+    // FIXME: filter duplicates
+
+    for (auto s : settings.trustedPublicKeys.get()) {
+        PublicKey key(s);
+        publicKeys.emplace(key.name, key);
+    }
+
+    for (auto secretKeyFile : settings.secretKeyFiles.get()) {
+        try {
+            SecretKey secretKey(readFile(secretKeyFile));
+            publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
+        } catch (SystemError & e) {
+            /* Ignore unreadable key files. That's normal in a
+               multi-user installation. */
+        }
+    }
+
+    return publicKeys;
+}
+
+}

src/libstore/keys.hh

@@ -0,0 +1,10 @@
+#pragma once
+///@file
+
+#include "signature/local-keys.hh"
+
+namespace nix {
+
+PublicKeys getDefaultPublicKeys();
+
+}

src/libstore/legacy-ssh-store.cc

@@ -22,45 +22,10 @@ std::string LegacySSHStoreConfig::doc()
 }
 
 
-struct LegacySSHStore::Connection
+struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection
 {
     std::unique_ptr<SSHMaster::Connection> sshConn;
-    FdSink to;
-    FdSource from;
-    ServeProto::Version remoteVersion;
     bool good = true;
-
-    /**
-     * Coercion to `ServeProto::ReadConn`. This makes it easy to use the
-     * factored out serve protocol searlizers with a
-     * `LegacySSHStore::Connection`.
-     *
-     * The serve protocol connection types are unidirectional, unlike
-     * this type.
-     */
-    operator ServeProto::ReadConn ()
-    {
-        return ServeProto::ReadConn {
-            .from = from,
-            .version = remoteVersion,
-        };
-    }
-
-    /*
-     * Coercion to `ServeProto::WriteConn`. This makes it easy to use the
-     * factored out serve protocol searlizers with a
-     * `LegacySSHStore::Connection`.
-     *
-     * The serve protocol connection types are unidirectional, unlike
-     * this type.
-     */
-    operator ServeProto::WriteConn ()
-    {
-        return ServeProto::WriteConn {
-            .to = to,
-            .version = remoteVersion,
-        };
-    }
 };
 
 
@@ -90,34 +55,31 @@ LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & h
 ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
 {
     auto conn = make_ref<Connection>();
-    conn->sshConn = master.startCommand(
-        fmt("%s --serve --write", remoteProgram)
-        + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
+    Strings command = remoteProgram.get();
+    command.push_back("--serve");
+    command.push_back("--write");
+    if (remoteStore.get() != "") {
+        command.push_back("--store");
+        command.push_back(remoteStore.get());
+    }
+    conn->sshConn = master.startCommand(std::move(command));
     conn->to = FdSink(conn->sshConn->in.get());
     conn->from = FdSource(conn->sshConn->out.get());
 
+    StringSink saved;
+    TeeSource tee(conn->from, saved);
     try {
-        conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
-        conn->to.flush();
-
-        StringSink saved;
-        try {
-            TeeSource tee(conn->from, saved);
-            unsigned int magic = readInt(tee);
-            if (magic != SERVE_MAGIC_2)
-                throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
-        } catch (SerialisationError & e) {
-            /* In case the other side is waiting for our input,
-               close it. */
-            conn->sshConn->in.close();
-            auto msg = conn->from.drain();
-            throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
-                host, chomp(saved.s + msg));
+        conn->remoteVersion = ServeProto::BasicClientConnection::handshake(
+            conn->to, tee, SERVE_PROTOCOL_VERSION, host);
+    } catch (SerialisationError & e) {
+        // in.close(): Don't let the remote block on us not writing.
+        conn->sshConn->in.close();
+        {
+            NullSink nullSink;
+            conn->from.drainInto(nullSink);
         }
-        conn->remoteVersion = readInt(conn->from);
-        if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
-            throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
-
+        throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
+            host, chomp(saved.s));
     } catch (EndOfFile & e) {
         throw Error("cannot connect to '%1%'", host);
     }
@@ -232,16 +194,16 @@ void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
 }
 
 
-void LegacySSHStore::putBuildSettings(Connection & conn)
+static ServeProto::BuildOptions buildSettings()
 {
-    ServeProto::write(*this, conn, ServeProto::BuildOptions {
+    return {
         .maxSilentTime = settings.maxSilentTime,
         .buildTimeout = settings.buildTimeout,
         .maxLogSize = settings.maxLogSize,
         .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
         .enforceDeterminism = 0,
         .keepFailed = settings.keepFailed,
-    });
+    };
 }
 
 
@@ -250,14 +212,7 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas
 {
     auto conn(connections->get());
 
-    conn->to
-        << ServeProto::Command::BuildDerivation
-        << printStorePath(drvPath);
-    writeDerivation(conn->to, *this, drv);
-
-    putBuildSettings(*conn);
-
-    conn->to.flush();
+    conn->putBuildDerivationRequest(*this, drvPath, drv, buildSettings());
 
     return ServeProto::Serialise<BuildResult>::read(*this, *conn);
 }
@@ -288,7 +243,7 @@ void LegacySSHStore::buildPaths(const std::vector<DerivedPath> & drvPaths, Build
     }
     conn->to << ss;
 
-    putBuildSettings(*conn);
+    ServeProto::write(*this, *conn, buildSettings());
 
     conn->to.flush();
 
@@ -328,15 +283,8 @@ StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
     SubstituteFlag maybeSubstitute)
 {
     auto conn(connections->get());
-
-    conn->to
-        << ServeProto::Command::QueryValidPaths
-        << false // lock
-        << maybeSubstitute;
-    ServeProto::write(*this, *conn, paths);
-    conn->to.flush();
-
-    return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
+    return conn->queryValidPaths(*this,
+        false, paths, maybeSubstitute);
 }
 
 

src/libstore/legacy-ssh-store.hh

@@ -13,7 +13,7 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
 {
     using CommonSSHStoreConfig::CommonSSHStoreConfig;
 
-    const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
+    const Setting<Strings> remoteProgram{this, {"nix-store"}, "remote-program",
         "Path to the `nix-store` executable on the remote machine."};
 
     const Setting<int> maxConnections{this, 1, "max-connections",
@@ -59,25 +59,24 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     { unsupported("queryPathFromHashPart"); }
 
     StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            PathFilter & filter,
-            RepairFlag repair,
-            const StorePathSet & references) override
+        std::string_view name,
+        SourceAccessor & accessor,
+        const CanonPath & srcPath,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        PathFilter & filter,
+        RepairFlag repair) override
     { unsupported("addToStore"); }
 
-    StorePath addTextToStore(
+    virtual StorePath addToStoreFromDump(
+        Source & dump,
         std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override
-    { unsupported("addTextToStore"); }
-
-private:
-
-    void putBuildSettings(Connection & conn);
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        RepairFlag repair = NoRepair) override
+    { unsupported("addToStore"); }
 
 public:
 

src/libstore/local-store.cc

@@ -13,11 +13,15 @@
 #include "compression.hh"
 #include "signals.hh"
 #include "posix-fs-canonicalise.hh"
+#include "posix-source-accessor.hh"
+#include "keys.hh"
 
 #include <iostream>
 #include <algorithm>
 #include <cstring>
 
+#include <memory>
+#include <new>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/select.h>
@@ -272,7 +276,7 @@ LocalStore::LocalStore(const Params & params)
                 [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize);
             }
         }
-    } catch (SysError & e) { /* don't care about errors */
+    } catch (SystemError & e) { /* don't care about errors */
     }
 
     /* Acquire the big fat lock in shared mode to make sure that no
@@ -1050,8 +1054,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     bool narRead = false;
     Finally cleanup = [&]() {
         if (!narRead) {
-            NullParseSink sink;
-            parseDump(sink, source);
+            NullFileSystemObjectSink sink;
+            try {
+                parseDump(sink, source);
+            } catch (...) {
+                ignoreException();
+            }
         }
     };
 
@@ -1094,11 +1102,22 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             if (info.ca) {
                 auto & specified = *info.ca;
-                auto actualHash = hashCAPath(
-                    specified.method,
-                    specified.hash.algo,
-                    info.path
-                );
+                auto actualHash = ({
+                    HashModuloSink caSink {
+                        specified.hash.algo,
+                        std::string { info.path.hashPart() },
+                    };
+                    PosixSourceAccessor accessor;
+                    dumpPath(
+                        *getFSAccessor(false),
+                        CanonPath { printStorePath(info.path) },
+                        caSink,
+                        specified.method.getFileIngestionMethod());
+                    ContentAddress {
+                        .method = specified.method,
+                        .hash = caSink.finish().first,
+                    };
+                });
                 if (specified.hash != actualHash.hash) {
                     throw Error("ca hash mismatch importing path '%s';\n  specified: %s\n  got:       %s",
                         printStorePath(info.path),
@@ -1121,8 +1140,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 }
 
 
-StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name,
-                                         FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
+StorePath LocalStore::addToStoreFromDump(
+    Source & source0,
+    std::string_view name,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     /* For computing the store path. */
     auto hashSink = std::make_unique<HashSink>(hashAlgo);
@@ -1136,7 +1160,11 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
        path. */
     bool inMemory = false;
 
-    std::string dump;
+    struct Free {
+        void operator()(void* v) { free(v); }
+    };
+    std::unique_ptr<char, Free> dumpBuffer(nullptr);
+    std::string_view dump;
 
     /* Fill out buffer, and decide whether we are working strictly in
        memory based on whether we break out because the buffer is full
@@ -1145,13 +1173,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
         auto oldSize = dump.size();
         constexpr size_t chunkSize = 65536;
         auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
-        dump.resize(oldSize + want);
+        if (auto tmp = realloc(dumpBuffer.get(), oldSize + want)) {
+            dumpBuffer.release();
+            dumpBuffer.reset((char*) tmp);
+        } else {
+            throw std::bad_alloc();
+        }
         auto got = 0;
         Finally cleanup([&]() {
-            dump.resize(oldSize + got);
+            dump = {dumpBuffer.get(), dump.size() + got};
         });
         try {
-            got = source.read(dump.data() + oldSize, want);
+            got = source.read(dumpBuffer.get() + oldSize, want);
         } catch (EndOfFile &) {
             inMemory = true;
             break;
@@ -1172,25 +1205,22 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
         delTempDir = std::make_unique<AutoDelete>(tempDir);
         tempPath = tempDir + "/x";
 
-        if (method == FileIngestionMethod::Recursive)
-            restorePath(tempPath, bothSource);
-        else
-            writeFile(tempPath, bothSource);
+        restorePath(tempPath, bothSource, method.getFileIngestionMethod());
 
-        dump.clear();
+        dumpBuffer.reset();
+        dump = {};
     }
 
     auto [hash, size] = hashSink->finish();
 
-    ContentAddressWithReferences desc = FixedOutputInfo {
-        .method = method,
-        .hash = hash,
-        .references = {
+    auto desc = ContentAddressWithReferences::fromParts(
+        method,
+        hash,
+        {
             .others = references,
             // caller is not capable of creating a self-reference, because this is content-addressed without modulus
             .self = false,
-        },
-    };
+        });
 
     auto dstPath = makeFixedOutputPathFromCA(name, desc);
 
@@ -1213,11 +1243,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
 
             if (inMemory) {
                 StringSource dumpSource { dump };
-                /* Restore from the NAR in memory. */
-                if (method == FileIngestionMethod::Recursive)
-                    restorePath(realPath, dumpSource);
-                else
-                    writeFile(realPath, dumpSource);
+                /* Restore from the buffer in memory. */
+                restorePath(realPath, dumpSource, method.getFileIngestionMethod());
             } else {
                 /* Move the temporary path we restored above. */
                 moveFile(tempPath, realPath);
@@ -1253,58 +1280,6 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
 }
 
 
-StorePath LocalStore::addTextToStore(
-    std::string_view name,
-    std::string_view s,
-    const StorePathSet & references, RepairFlag repair)
-{
-    auto hash = hashString(HashAlgorithm::SHA256, s);
-    auto dstPath = makeTextPath(name, TextInfo {
-        .hash = hash,
-        .references = references,
-    });
-
-    addTempRoot(dstPath);
-
-    if (repair || !isValidPath(dstPath)) {
-
-        auto realPath = Store::toRealPath(dstPath);
-
-        PathLocks outputLock({realPath});
-
-        if (repair || !isValidPath(dstPath)) {
-
-            deletePath(realPath);
-
-            autoGC();
-
-            writeFile(realPath, s);
-
-            canonicalisePathMetaData(realPath, {});
-
-            StringSink sink;
-            dumpString(s, sink);
-            auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
-
-            optimisePath(realPath, repair);
-
-            ValidPathInfo info { dstPath, narHash };
-            info.narSize = sink.s.size();
-            info.references = references;
-            info.ca = {
-                .method = TextIngestionMethod {},
-                .hash = hash,
-            };
-            registerValidPath(info);
-        }
-
-        outputLock.setDeletion(true);
-    }
-
-    return dstPath;
-}
-
-
 /* Create a temporary directory in the store that won't be
    garbage-collected until the returned FD is closed. */
 std::pair<Path, AutoCloseFD> LocalStore::createTempDirInStore()
@@ -1367,7 +1342,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
         for (auto & link : readDirectory(linksDir)) {
             printMsg(lvlTalkative, "checking contents of '%s'", link.name);
             Path linkPath = linksDir + "/" + link.name;
-            std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false);
+            PosixSourceAccessor accessor;
+            std::string hash = hashPath(
+                accessor, CanonPath { linkPath },
+                FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false);
             if (hash != link.name) {
                 printError("link '%s' was modified! expected hash '%s', got '%s'",
                     linkPath, link.name, hash);
@@ -1623,7 +1601,8 @@ void LocalStore::signRealisation(Realisation & realisation)
 
     for (auto & secretKeyFile : secretKeyFiles.get()) {
         SecretKey secretKey(readFile(secretKeyFile));
-        realisation.sign(secretKey);
+        LocalSigner signer(std::move(secretKey));
+        realisation.sign(signer);
     }
 }
 
@@ -1635,7 +1614,8 @@ void LocalStore::signPathInfo(ValidPathInfo & info)
 
     for (auto & secretKeyFile : secretKeyFiles.get()) {
         SecretKey secretKey(readFile(secretKeyFile));
-        info.sign(*this, secretKey);
+        LocalSigner signer(std::move(secretKey));
+        info.sign(*this, signer);
     }
 }
 
@@ -1714,42 +1694,6 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id,
     }
 }
 
-ContentAddress LocalStore::hashCAPath(
-    const ContentAddressMethod & method, const HashAlgorithm & hashAlgo,
-    const StorePath & path)
-{
-    return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart());
-}
-
-ContentAddress LocalStore::hashCAPath(
-    const ContentAddressMethod & method,
-    const HashAlgorithm & hashAlgo,
-    const Path & path,
-    const std::string_view pathHash
-)
-{
-    HashModuloSink caSink ( hashAlgo, std::string(pathHash) );
-    std::visit(overloaded {
-        [&](const TextIngestionMethod &) {
-            readFile(path, caSink);
-        },
-        [&](const FileIngestionMethod & m2) {
-            switch (m2) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(path, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(path, caSink);
-                break;
-            }
-        },
-    }, method.raw);
-    return ContentAddress {
-        .method = method,
-        .hash = caSink.finish().first,
-    };
-}
-
 void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
 {
     assert(drvPath.isDerivation());

src/libstore/local-store.hh

@@ -177,12 +177,11 @@ public:
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStoreFromDump(Source & dump, std::string_view name,
-                                 FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override;
-
-    StorePath addTextToStore(
+    StorePath addToStoreFromDump(
+        Source & dump,
         std::string_view name,
-        std::string_view s,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
         const StorePathSet & references,
         RepairFlag repair) override;
 
@@ -396,19 +395,6 @@ private:
     void signPathInfo(ValidPathInfo & info);
     void signRealisation(Realisation &);
 
-    // XXX: Make a generic `Store` method
-    ContentAddress hashCAPath(
-        const ContentAddressMethod & method,
-        const HashAlgorithm & hashAlgo,
-        const StorePath & path);
-
-    ContentAddress hashCAPath(
-        const ContentAddressMethod & method,
-        const HashAlgorithm & hashAlgo,
-        const Path & path,
-        const std::string_view pathHash
-    );
-
     void addBuildLog(const StorePath & drvPath, std::string_view log) override;
 
     friend struct LocalDerivationGoal;

src/libstore/local.mk

@@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 
 libstore_LIBS = libutil
 
-libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
+libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
  libstore_LDFLAGS += -ldl
 endif
@@ -16,15 +16,15 @@ endif
 $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
 
 ifeq ($(ENABLE_S3), 1)
-	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp
+  libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp
 endif
 
 ifdef HOST_SOLARIS
-	libstore_LDFLAGS += -lsocket
+  libstore_LDFLAGS += -lsocket
 endif
 
 ifeq ($(HAVE_SECCOMP), 1)
-	libstore_LDFLAGS += $(LIBSECCOMP_LIBS)
+  libstore_LDFLAGS += $(LIBSECCOMP_LIBS)
 endif
 
 libstore_CXXFLAGS += \
@@ -48,9 +48,9 @@ $(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell)
 	$(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp
 	@mv $@.tmp $@
 else
-ifneq ($(sandbox_shell),)
-libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
-endif
+  ifneq ($(sandbox_shell),)
+    libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
+  endif
 endif
 
 $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh

src/libstore/machines.cc

@@ -32,11 +32,19 @@ Machine::Machine(decltype(storeUri) storeUri,
     systemTypes(systemTypes),
     sshKey(sshKey),
     maxJobs(maxJobs),
-    speedFactor(std::max(1U, speedFactor)),
+    speedFactor(speedFactor == 0.0f ? 1.0f : std::move(speedFactor)),
     supportedFeatures(supportedFeatures),
     mandatoryFeatures(mandatoryFeatures),
     sshPublicHostKey(sshPublicHostKey)
-{}
+{
+    if (speedFactor < 0.0)
+        throw UsageError("speed factor must be >= 0");
+}
+
+bool Machine::systemSupported(const std::string & system) const
+{
+    return system == "builtin" || (systemTypes.count(system) > 0);
+}
 
 bool Machine::allSupported(const std::set<std::string> & features) const
 {
@@ -130,6 +138,14 @@ static Machine parseBuilderLine(const std::string & line)
         return result.value();
     };
 
+    auto parseFloatField = [&](size_t fieldIndex) {
+        const auto result = string2Int<float>(tokens[fieldIndex]);
+        if (!result) {
+            throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'float'", fieldIndex, line);
+        }
+        return result.value();
+    };
+
     auto ensureBase64 = [&](size_t fieldIndex) {
         const auto & str = tokens[fieldIndex];
         try {
@@ -145,10 +161,10 @@ static Machine parseBuilderLine(const std::string & line)
 
     return {
         tokens[0],
-        isSet(1) ? tokenizeString<std::vector<std::string>>(tokens[1], ",") : std::vector<std::string>{settings.thisSystem},
+        isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",") : std::set<std::string>{settings.thisSystem},
         isSet(2) ? tokens[2] : "",
         isSet(3) ? parseUnsignedIntField(3) : 1U,
-        isSet(4) ? parseUnsignedIntField(4) : 1U,
+        isSet(4) ? parseFloatField(4) : 1.0f,
         isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{},
         isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{},
         isSet(7) ? ensureBase64(7) : ""

src/libstore/machines.hh

@@ -10,17 +10,30 @@ class Store;
 struct Machine {
 
     const std::string storeUri;
-    const std::vector<std::string> systemTypes;
+    const std::set<std::string> systemTypes;
     const std::string sshKey;
     const unsigned int maxJobs;
-    const unsigned int speedFactor;
+    const float speedFactor;
     const std::set<std::string> supportedFeatures;
     const std::set<std::string> mandatoryFeatures;
     const std::string sshPublicHostKey;
     bool enabled = true;
 
+    /**
+     * @return Whether `system` is either `"builtin"` or in
+     * `systemTypes`.
+     */
+    bool systemSupported(const std::string & system) const;
+
+    /**
+     * @return Whether `features` is a subset of the union of `supportedFeatures` and
+     * `mandatoryFeatures`
+     */
     bool allSupported(const std::set<std::string> & features) const;
 
+    /**
+     * @return @Whether `mandatoryFeatures` is a subset of `features`
+     */
     bool mandatoryMet(const std::set<std::string> & features) const;
 
     Machine(decltype(storeUri) storeUri,

src/libstore/misc.cc

@@ -331,8 +331,11 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
 std::map<DrvOutput, StorePath> drvOutputReferences(
     Store & store,
     const Derivation & drv,
-    const StorePath & outputPath)
+    const StorePath & outputPath,
+    Store * evalStore_)
 {
+    auto & evalStore = evalStore_ ? *evalStore_ : store;
+
     std::set<Realisation> inputRealisations;
 
     std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumRealisations;
@@ -340,7 +343,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
     accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
         if (!inputNode.value.empty()) {
             auto outputHashes =
-                staticOutputHashes(store, store.readDerivation(inputDrv));
+                staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
             for (const auto & outputName : inputNode.value) {
                 auto outputHash = get(outputHashes, outputName);
                 if (!outputHash)
@@ -362,7 +365,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
                 SingleDerivedPath next = SingleDerivedPath::Built { d, outputName };
                 accumRealisations(
                     // TODO deep resolutions for dynamic derivations, issue #8947, would go here.
-                    resolveDerivedPath(store, next),
+                    resolveDerivedPath(store, next, evalStore_),
                     childNode);
             }
         }

src/libstore/nar-accessor.cc

@@ -19,6 +19,35 @@ struct NarMember
     std::map<std::string, NarMember> children;
 };
 
+struct NarMemberConstructor : CreateRegularFileSink
+{
+private:
+
+    NarMember & narMember;
+
+    uint64_t & pos;
+
+public:
+
+    NarMemberConstructor(NarMember & nm, uint64_t & pos)
+        : narMember(nm), pos(pos)
+    { }
+
+    void isExecutable() override
+    {
+        narMember.stat.isExecutable = true;
+    }
+
+    void preallocateContents(uint64_t size) override
+    {
+        narMember.stat.fileSize = size;
+        narMember.stat.narOffset = pos;
+    }
+
+    void operator () (std::string_view data) override
+    { }
+};
+
 struct NarAccessor : public SourceAccessor
 {
     std::optional<const std::string> nar;
@@ -27,7 +56,7 @@ struct NarAccessor : public SourceAccessor
 
     NarMember root;
 
-    struct NarIndexer : ParseSink, Source
+    struct NarIndexer : FileSystemObjectSink, Source
     {
         NarAccessor & acc;
         Source & source;
@@ -42,7 +71,7 @@ struct NarAccessor : public SourceAccessor
             : acc(acc), source(source)
         { }
 
-        void createMember(const Path & path, NarMember member)
+        NarMember & createMember(const Path & path, NarMember member)
         {
             size_t level = std::count(path.begin(), path.end(), '/');
             while (parents.size() > level) parents.pop();
@@ -50,11 +79,14 @@ struct NarAccessor : public SourceAccessor
             if (parents.empty()) {
                 acc.root = std::move(member);
                 parents.push(&acc.root);
+                return acc.root;
             } else {
                 if (parents.top()->stat.type != Type::tDirectory)
                     throw Error("NAR file missing parent directory of path '%s'", path);
                 auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
-                parents.push(&result.first->second);
+                auto & ref = result.first->second;
+                parents.push(&ref);
+                return ref;
             }
         }
 
@@ -68,34 +100,18 @@ struct NarAccessor : public SourceAccessor
             } });
         }
 
-        void createRegularFile(const Path & path) override
+        void createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func) override
         {
-            createMember(path, NarMember{ .stat = {
+            auto & nm = createMember(path, NarMember{ .stat = {
                 .type = Type::tRegular,
                 .fileSize = 0,
                 .isExecutable = false,
                 .narOffset = 0
             } });
+            NarMemberConstructor nmc { nm, pos };
+            func(nmc);
         }
 
-        void closeRegularFile() override
-        { }
-
-        void isExecutable() override
-        {
-            parents.top()->stat.isExecutable = true;
-        }
-
-        void preallocateContents(uint64_t size) override
-        {
-            auto & st = parents.top()->stat;
-            st.fileSize = size;
-            st.narOffset = pos;
-        }
-
-        void receiveContents(std::string_view data) override
-        { }
-
         void createSymlink(const Path & path, const std::string & target) override
         {
             createMember(path,

src/libstore/optimise-store.cc

@@ -2,6 +2,7 @@
 #include "globals.hh"
 #include "signals.hh"
 #include "posix-fs-canonicalise.hh"
+#include "posix-source-accessor.hh"
 
 #include <cstdlib>
 #include <cstring>
@@ -146,7 +147,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        Also note that if `path' is a symlink, then we're hashing the
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
-    Hash hash = hashPath(HashAlgorithm::SHA256, path).first;
+    Hash hash = ({
+        PosixSourceAccessor accessor;
+        hashPath(
+            accessor, CanonPath { path },
+            FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
+    });
     debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
 
     /* Check if this is a known hash. */
@@ -156,7 +162,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     if (pathExists(linkPath)) {
         auto stLink = lstat(linkPath);
         if (st.st_size != stLink.st_size
-            || (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first))
+            || (repair && hash != ({
+                PosixSourceAccessor accessor;
+                hashPath(
+                    accessor, CanonPath { linkPath },
+                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
+           })))
         {
             // XXX: Consider overwriting linkPath with our valid version.
             warn("removing corrupted link '%s'", linkPath);
@@ -231,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Atomically replace the old file with the new hard link. */
     try {
         renameFile(tempLink, path);
-    } catch (SysError & e) {
+    } catch (SystemError & e) {
         if (unlink(tempLink.c_str()) == -1)
             printError("unable to unlink '%1%'", tempLink);
         if (errno == EMLINK) {

src/libstore/path-info.cc

@@ -38,9 +38,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
 }
 
 
-void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
+void ValidPathInfo::sign(const Store & store, const Signer & signer)
 {
-    sigs.insert(secretKey.signDetached(fingerprint(store)));
+    sigs.insert(signer.signDetached(fingerprint(store)));
 }
 
 std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferences() const

src/libstore/path-info.hh

@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "crypto.hh"
+#include "signature/signer.hh"
 #include "path.hh"
 #include "hash.hh"
 #include "content-address.hh"
@@ -107,7 +107,7 @@ struct ValidPathInfo : UnkeyedValidPathInfo {
      */
     std::string fingerprint(const Store & store) const;
 
-    void sign(const Store & store, const SecretKey & secretKey);
+    void sign(const Store & store, const Signer & signer);
 
     /**
      * @return The `ContentAddressWithReferences` that determines the

src/libstore/path-regex.hh

@@ -3,6 +3,11 @@
 
 namespace nix {
 
-static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)";
+
+static constexpr std::string_view nameRegexStr =
+    // This uses a negative lookahead: (?!\.\.?(-|$))
+    //   - deny ".", "..", or those strings followed by '-'
+    //   - when it's not those, start again at the start of the input and apply the next regex, which is [0-9a-zA-Z\+\-\._\?=]+
+    R"((?!\.\.?(-|$))[0-9a-zA-Z\+\-\._\?=]+)";
 
 }

src/libstore/path.cc

@@ -1,7 +1,5 @@
 #include "store-dir-config.hh"
 
-#include <sodium.h>
-
 namespace nix {
 
 static void checkName(std::string_view path, std::string_view name)
@@ -11,9 +9,20 @@ static void checkName(std::string_view path, std::string_view name)
     if (name.size() > StorePath::MaxPathLen)
         throw BadStorePath("store path '%s' has a name longer than %d characters",
             path, StorePath::MaxPathLen);
-    if (name[0] == '.')
-        throw BadStorePath("store path '%s' starts with illegal character '.'", path);
     // See nameRegexStr for the definition
+    if (name[0] == '.') {
+        // check against "." and "..", followed by end or dash
+        if (name.size() == 1)
+            throw BadStorePath("store path '%s' has invalid name '%s'", path, name);
+        if (name[1] == '-')
+            throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, ".");
+        if (name[1] == '.') {
+            if (name.size() == 2)
+                throw BadStorePath("store path '%s' has invalid name '%s'", path, name);
+            if (name[2] == '-')
+                throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, "..");
+        }
+    }
     for (auto c : name)
         if (!((c >= '0' && c <= '9')
                 || (c >= 'a' && c <= 'z')
@@ -49,9 +58,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
 
 StorePath StorePath::random(std::string_view name)
 {
-    Hash hash(HashAlgorithm::SHA1);
-    randombytes_buf(hash.hash, hash.hashSize);
-    return StorePath(hash, name);
+    return StorePath(Hash::random(HashAlgorithm::SHA1), name);
 }
 
 StorePath StoreDirConfig::parseStorePath(std::string_view path) const

src/libstore/posix-fs-canonicalise.cc

@@ -1,4 +1,4 @@
-#if HAVE_SYS_XATTR_H
+#if HAVE_ACL_SUPPORT
 # include <sys/xattr.h>
 #endif
 
@@ -78,7 +78,7 @@ static void canonicalisePathMetaData_(
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
         throw Error("file '%1%' has an unsupported type", path);
 
-#ifdef HAVE_SYS_XATTR_H
+#if HAVE_ACL_SUPPORT
     /* Remove extended attributes / ACLs. */
     ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
 

src/libstore/realisation.cc

@@ -1,6 +1,7 @@
 #include "realisation.hh"
 #include "store-api.hh"
 #include "closure.hh"
+#include "signature/local-keys.hh"
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -113,9 +114,9 @@ std::string Realisation::fingerprint() const
     return serialized.dump();
 }
 
-void Realisation::sign(const SecretKey & secretKey)
+void Realisation::sign(const Signer &signer)
 {
-    signatures.insert(secretKey.signDetached(fingerprint()));
+    signatures.insert(signer.signDetached(fingerprint()));
 }
 
 bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const

src/libstore/realisation.hh

@@ -8,7 +8,7 @@
 #include "derived-path.hh"
 #include <nlohmann/json_fwd.hpp>
 #include "comparator.hh"
-#include "crypto.hh"
+#include "signature/signer.hh"
 
 namespace nix {
 
@@ -64,7 +64,7 @@ struct Realisation {
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
 
     std::string fingerprint() const;
-    void sign(const SecretKey &);
+    void sign(const Signer &);
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
     size_t checkSignatures(const PublicKeys & publicKeys) const;
 

src/libstore/remote-fs-accessor.cc

@@ -87,13 +87,13 @@ std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPat
             nars.emplace(storePath.hashPart(), narAccessor);
             return {narAccessor, restPath};
 
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
 
         try {
             auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
             nars.emplace(storePath.hashPart(), narAccessor);
             return {narAccessor, restPath};
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
     }
 
     StringSink sink;

src/libstore/remote-store.cc

@@ -16,6 +16,8 @@
 #include "logging.hh"
 #include "callback.hh"
 #include "filetransfer.hh"
+#include "signals.hh"
+
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -65,6 +67,7 @@ void RemoteStore::initConnection(Connection & conn)
 {
     /* Send the magic greeting, check for the reply. */
     try {
+        conn.from.endOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)";
         conn.to << WORKER_MAGIC_1;
         conn.to.flush();
         StringSink saved;
@@ -186,7 +189,7 @@ void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source,
                 if (m.find("parsing derivation") != std::string::npos &&
                     m.find("expected string") != std::string::npos &&
                     m.find("Derive([") != std::string::npos)
-                    throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
+                    throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
             }
             throw;
         }
@@ -225,7 +228,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         conn->to << WorkerProto::Op::QueryValidPaths;
         WorkerProto::write(*this, *conn, paths);
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
-            conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
+            conn->to << maybeSubstitute;
         }
         conn.processStderr();
         return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
@@ -502,8 +505,13 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 }
 
 
-StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
-                                          FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
+StorePath RemoteStore::addToStoreFromDump(
+    Source & dump,
+    std::string_view name,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     return addCAToStore(dump, name, method, hashAlgo, references, repair)->path;
 }
@@ -603,16 +611,6 @@ void RemoteStore::addMultipleToStore(
 }
 
 
-StorePath RemoteStore::addTextToStore(
-    std::string_view name,
-    std::string_view s,
-    const StorePathSet & references,
-    RepairFlag repair)
-{
-    StringSource source(s);
-    return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path;
-}
-
 void RemoteStore::registerDrvOutput(const Realisation & info)
 {
     auto conn(getConnection());
@@ -1071,6 +1069,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sin
     std::thread stderrThread([&]()
     {
         try {
+            ReceiveInterrupts receiveInterrupts;
             processStderr(nullptr, nullptr, false);
         } catch (...) {
             ex = std::current_exception();

src/libstore/remote-store.hh

@@ -82,10 +82,15 @@ public:
             RepairFlag repair);
 
     /**
-     * Add a content-addressable store path. Does not support references. `dump` will be drained.
+     * Add a content-addressable store path. `dump` will be drained.
      */
-    StorePath addToStoreFromDump(Source & dump, std::string_view name,
-                                 FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
+    StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        RepairFlag repair = NoRepair) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
@@ -101,12 +106,6 @@ public:
         RepairFlag repair,
         CheckSigsFlag checkSigs) override;
 
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override;
-
     void registerDrvOutput(const Realisation & info) override;
 
     void queryRealisationUncached(const DrvOutput &,

src/libstore/repair-flag.hh

@@ -1,8 +0,0 @@
-#pragma once
-///@file
-
-namespace nix {
-
-enum RepairFlag : bool { NoRepair = false, Repair = true };
-
-}

src/libstore/serve-protocol-impl.cc

@@ -0,0 +1,69 @@
+#include "serve-protocol-impl.hh"
+#include "build-result.hh"
+#include "derivations.hh"
+
+namespace nix {
+
+ServeProto::Version ServeProto::BasicClientConnection::handshake(
+    BufferedSink & to,
+    Source & from,
+    ServeProto::Version localVersion,
+    std::string_view host)
+{
+    to << SERVE_MAGIC_1 << localVersion;
+    to.flush();
+
+    unsigned int magic = readInt(from);
+    if (magic != SERVE_MAGIC_2)
+        throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
+    auto remoteVersion = readInt(from);
+    if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
+        throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
+    return remoteVersion;
+}
+
+ServeProto::Version ServeProto::BasicServerConnection::handshake(
+    BufferedSink & to,
+    Source & from,
+    ServeProto::Version localVersion)
+{
+    unsigned int magic = readInt(from);
+    if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch");
+    to << SERVE_MAGIC_2 << localVersion;
+    to.flush();
+    return readInt(from);
+}
+
+
+StorePathSet ServeProto::BasicClientConnection::queryValidPaths(
+    const Store & store,
+    bool lock, const StorePathSet & paths,
+    SubstituteFlag maybeSubstitute)
+{
+    to
+        << ServeProto::Command::QueryValidPaths
+        << lock
+        << maybeSubstitute;
+    write(store, *this, paths);
+    to.flush();
+
+    return Serialise<StorePathSet>::read(store, *this);
+}
+
+
+void ServeProto::BasicClientConnection::putBuildDerivationRequest(
+    const Store & store,
+    const StorePath & drvPath, const BasicDerivation & drv,
+    const ServeProto::BuildOptions & options)
+{
+    to
+        << ServeProto::Command::BuildDerivation
+        << store.printStorePath(drvPath);
+    writeDerivation(to, store, drv);
+
+    ServeProto::write(store, *this, options);
+
+    to.flush();
+}
+
+}

src/libstore/serve-protocol-impl.hh

@@ -10,6 +10,7 @@
 
 #include "serve-protocol.hh"
 #include "length-prefixed-protocol-helper.hh"
+#include "store-api.hh"
 
 namespace nix {
 
@@ -56,4 +57,101 @@ struct ServeProto::Serialise
 
 /* protocol-specific templates */
 
+struct ServeProto::BasicClientConnection
+{
+    FdSink to;
+    FdSource from;
+    ServeProto::Version remoteVersion;
+
+    /**
+     * Establishes connection, negotiating version.
+     *
+     * @return the version provided by the other side of the
+     * connection.
+     *
+     * @param to Taken by reference to allow for various error handling
+     * mechanisms.
+     *
+     * @param from Taken by reference to allow for various error
+     * handling mechanisms.
+     *
+     * @param localVersion Our version which is sent over
+     *
+     * @param host Just used to add context to thrown exceptions.
+     */
+    static ServeProto::Version handshake(
+        BufferedSink & to,
+        Source & from,
+        ServeProto::Version localVersion,
+        std::string_view host);
+
+    /**
+     * Coercion to `ServeProto::ReadConn`. This makes it easy to use the
+     * factored out serve protocol serializers with a
+     * `LegacySSHStore::Connection`.
+     *
+     * The serve protocol connection types are unidirectional, unlike
+     * this type.
+     */
+    operator ServeProto::ReadConn ()
+    {
+        return ServeProto::ReadConn {
+            .from = from,
+            .version = remoteVersion,
+        };
+    }
+
+    /**
+     * Coercion to `ServeProto::WriteConn`. This makes it easy to use the
+     * factored out serve protocol serializers with a
+     * `LegacySSHStore::Connection`.
+     *
+     * The serve protocol connection types are unidirectional, unlike
+     * this type.
+     */
+    operator ServeProto::WriteConn ()
+    {
+        return ServeProto::WriteConn {
+            .to = to,
+            .version = remoteVersion,
+        };
+    }
+
+    StorePathSet queryValidPaths(
+        const Store & remoteStore,
+        bool lock, const StorePathSet & paths,
+        SubstituteFlag maybeSubstitute);
+
+    /**
+     * Just the request half, because Hydra may do other things between
+     * issuing the request and reading the `BuildResult` response.
+     */
+    void putBuildDerivationRequest(
+        const Store & store,
+        const StorePath & drvPath, const BasicDerivation & drv,
+        const ServeProto::BuildOptions & options);
+};
+
+struct ServeProto::BasicServerConnection
+{
+    /**
+     * Establishes connection, negotiating version.
+     *
+     * @return the version provided by the other side of the
+     * connection.
+     *
+     * @param to Taken by reference to allow for various error handling
+     * mechanisms.
+     *
+     * @param from Taken by reference to allow for various error
+     * handling mechanisms.
+     *
+     * @param localVersion Our version which is sent over
+     */
+    static ServeProto::Version handshake(
+        BufferedSink & to,
+        Source & from,
+        ServeProto::Version localVersion);
+};
+
 }

src/libstore/serve-protocol.hh

@@ -59,6 +59,14 @@ struct ServeProto
         Version version;
     };
 
+    /**
+     * Stripped down serialization logic suitable for sharing with Hydra.
+     *
+     * @todo remove once Hydra uses Store abstraction consistently.
+     */
+    struct BasicClientConnection;
+    struct BasicServerConnection;
+
     /**
      * Data type for canonical pairs of serialisers for the serve protocol.
      *

src/libstore/ssh-store.cc

@@ -17,7 +17,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
     using RemoteStoreConfig::RemoteStoreConfig;
     using CommonSSHStoreConfig::CommonSSHStoreConfig;
 
-    const Setting<Path> remoteProgram{this, "nix-daemon", "remote-program",
+    const Setting<Strings> remoteProgram{this, {"nix-daemon"}, "remote-program",
         "Path to the `nix-daemon` executable on the remote machine."};
 
     const std::string name() override { return "Experimental SSH Store"; }
@@ -212,14 +212,15 @@ public:
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     auto conn = make_ref<Connection>();
-
-    std::string command = remoteProgram + " --stdio";
-    if (remoteStore.get() != "")
-        command += " --store " + shellEscape(remoteStore.get());
-    for (auto & arg : extraRemoteProgramArgs)
-        command += " " + shellEscape(arg);
-
-    conn->sshConn = master.startCommand(command);
+    Strings command = remoteProgram.get();
+    command.push_back("--stdio");
+    if (remoteStore.get() != "") {
+        command.push_back("--store");
+        command.push_back(remoteStore.get());
+    }
+    command.insert(command.end(),
+        extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end());
+    conn->sshConn = master.startCommand(std::move(command));
     conn->to = FdSink(conn->sshConn->in.get());
     conn->from = FdSource(conn->sshConn->out.get());
     return conn;

src/libstore/ssh.cc

@@ -52,7 +52,8 @@ bool SSHMaster::isMasterRunning() {
     return res.first == 0;
 }
 
-std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
+std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(
+    Strings && command, Strings && extraSshArgs)
 {
     Path socketPath = startMaster();
 
@@ -84,18 +85,19 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
 
         Strings args;
 
-        if (fakeSSH) {
-            args = { "bash", "-c" };
-        } else {
+        if (!fakeSSH) {
             args = { "ssh", host.c_str(), "-x" };
             addCommonSSHOpts(args);
             if (socketPath != "")
                 args.insert(args.end(), {"-S", socketPath});
             if (verbosity >= lvlChatty)
                 args.push_back("-v");
+            args.splice(args.end(), std::move(extraSshArgs));
+            args.push_back("--");
         }
 
-        args.push_back(command);
+        args.splice(args.end(), std::move(command));
+
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
         // could not exec ssh/bash

src/libstore/ssh.hh

@@ -41,7 +41,16 @@ public:
         AutoCloseFD out, in;
     };
 
-    std::unique_ptr<Connection> startCommand(const std::string & command);
+    /**
+     * @param command The command (arg vector) to execute.
+     *
+     * @param extraSShArgs Extra args to pass to SSH (not the command to
+     * execute). Will not be used when "fake SSHing" to the local
+     * machine.
+     */
+    std::unique_ptr<Connection> startCommand(
+        Strings && command,
+        Strings && extraSshArgs = {});
 
     Path startMaster();
 };

src/libstore/store-api.cc

@@ -1,4 +1,4 @@
-#include "crypto.hh"
+#include "signature/local-keys.hh"
 #include "source-accessor.hh"
 #include "globals.hh"
 #include "derived-path.hh"
@@ -194,7 +194,10 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed
     if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
     } else {
-        assert(info.references.size() == 0);
+        if (!info.references.empty()) {
+            throw Error("fixed output derivation '%s' is not allowed to refer to other store paths.\nYou may need to use the 'unsafeDiscardReferences' derivation attribute, see the manual for more details.",
+                name);
+        }
         return makeStorePath("output:out",
             hashString(HashAlgorithm::SHA256,
                 "fixed:out:"
@@ -205,25 +208,19 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed
 }
 
 
-StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const
-{
-    assert(info.hash.algo == HashAlgorithm::SHA256);
-    return makeStorePath(
-        makeType(*this, "text", StoreReferences {
-            .others = info.references,
-            .self = false,
-        }),
-        info.hash,
-        name);
-}
-
-
 StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
 {
     // New template
     return std::visit(overloaded {
         [&](const TextInfo & ti) {
-            return makeTextPath(name, ti);
+            assert(ti.hash.algo == HashAlgorithm::SHA256);
+            return makeStorePath(
+                makeType(*this, "text", StoreReferences {
+                    .others = ti.references,
+                    .self = false,
+                }),
+                ti.hash,
+                name);
         },
         [&](const FixedOutputInfo & foi) {
             return makeFixedOutputPath(name, foi);
@@ -232,54 +229,45 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const
 }
 
 
-std::pair<StorePath, Hash> StoreDirConfig::computeStorePathFromDump(
-        Source & dump,
-        std::string_view name,
-        FileIngestionMethod method,
-        HashAlgorithm hashAlgo,
-        const StorePathSet & references) const
-{
-    HashSink sink(hashAlgo);
-    dump.drainInto(sink);
-    auto h = sink.finish().first;
-    FixedOutputInfo caInfo {
-        .method = method,
-        .hash = h,
-        .references = {},
-    };
-    return std::make_pair(makeFixedOutputPath(name, caInfo), h);
-}
-
-
-StorePath StoreDirConfig::computeStorePathForText(
+std::pair<StorePath, Hash> StoreDirConfig::computeStorePath(
     std::string_view name,
-    std::string_view s,
-    const StorePathSet & references) const
+    SourceAccessor & accessor,
+    const CanonPath & path,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    PathFilter & filter) const
 {
-    return makeTextPath(name, TextInfo {
-        .hash = hashString(HashAlgorithm::SHA256, s),
-        .references = references,
-    });
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
+    return {
+        makeFixedOutputPathFromCA(
+            name,
+            ContentAddressWithReferences::fromParts(
+                method,
+                h,
+                {
+                    .others = references,
+                    .self = false,
+                })),
+        h,
+    };
 }
 
 
 StorePath Store::addToStore(
-        std::string_view name,
-        const Path & _srcPath,
-        FileIngestionMethod method,
-        HashAlgorithm hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references)
+    std::string_view name,
+    SourceAccessor & accessor,
+    const CanonPath & path,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    PathFilter & filter,
+    RepairFlag repair)
 {
-    Path srcPath(absPath(_srcPath));
     auto source = sinkToSource([&](Sink & sink) {
-        if (method == FileIngestionMethod::Recursive)
-            dumpPath(srcPath, sink, filter);
-        else
-            readFile(srcPath, sink);
+        dumpPath(accessor, path, sink, method.getFileIngestionMethod(), filter);
     });
-    return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
+    return addToStoreFromDump(*source, name, method, hashAlgo, references, repair);
 }
 
 void Store::addMultipleToStore(
@@ -404,9 +392,13 @@ digraph graphname {
     fileSink -> caHashSink
 }
 */
-ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
-                                    FileIngestionMethod method, HashAlgorithm hashAlgo,
-                                    std::optional<Hash> expectedCAHash)
+ValidPathInfo Store::addToStoreSlow(
+    std::string_view name,
+    SourceAccessor & accessor,
+    const CanonPath & srcPath,
+    ContentAddressMethod method, HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    std::optional<Hash> expectedCAHash)
 {
     HashSink narHashSink { HashAlgorithm::SHA256 };
     HashSink caHashSink { hashAlgo };
@@ -425,17 +417,19 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
        srcPath. The fact that we use scratchpadSink as a temporary buffer here
        is an implementation detail. */
     auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
-        dumpPath(srcPath, scratchpadSink);
+        accessor.dumpPath(srcPath, scratchpadSink);
     });
 
     /* tapped provides the same data as fileSource, but we also write all the
        information to narSink. */
     TeeSource tapped { *fileSource, narSink };
 
-    NullParseSink blank;
-    auto & parseSink = method == FileIngestionMethod::Flat
-        ? (ParseSink &) fileSink
-        : (ParseSink &) blank;
+    NullFileSystemObjectSink blank;
+    auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat
+        ? (FileSystemObjectSink &) fileSink
+        : method.getFileIngestionMethod() == FileIngestionMethod::Recursive
+        ? (FileSystemObjectSink &) blank
+        : (abort(), (FileSystemObjectSink &)*(FileSystemObjectSink *)nullptr); // handled both cases
 
     /* The information that flows from tapped (besides being replicated in
        narSink), is now put in parseSink. */
@@ -452,21 +446,24 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     if (expectedCAHash && expectedCAHash != hash)
         throw Error("hash mismatch for '%s'", srcPath);
 
+
     ValidPathInfo info {
         *this,
         name,
-        FixedOutputInfo {
-            .method = method,
-            .hash = hash,
-            .references = {},
-        },
+        ContentAddressWithReferences::fromParts(
+            method,
+            hash,
+            {
+                .others = references,
+                .self = false,
+            }),
         narHash,
     };
     info.narSize = narSize;
 
     if (!isValidPath(info.path)) {
         auto source = sinkToSource([&](Sink & scratchpadSink) {
-            dumpPath(srcPath, scratchpadSink);
+            accessor.dumpPath(srcPath, scratchpadSink);
         });
         addToStore(info, *source);
     }
@@ -689,38 +686,56 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual)
 }
 
 
+std::optional<std::shared_ptr<const ValidPathInfo>> Store::queryPathInfoFromClientCache(const StorePath & storePath)
+{
+    auto hashPart = std::string(storePath.hashPart());
+
+    {
+        auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
+        if (res && res->isKnownNow()) {
+            stats.narInfoReadAverted++;
+            if (res->didExist())
+                return std::make_optional(res->value);
+            else
+                return std::make_optional(nullptr);
+        }
+    }
+
+    if (diskCache) {
+        auto res = diskCache->lookupNarInfo(getUri(), hashPart);
+        if (res.first != NarInfoDiskCache::oUnknown) {
+            stats.narInfoReadAverted++;
+            {
+                auto state_(state.lock());
+                state_->pathInfoCache.upsert(std::string(storePath.to_string()),
+                    res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
+                if (res.first == NarInfoDiskCache::oInvalid ||
+                    !goodStorePath(storePath, res.second->path))
+                    return std::make_optional(nullptr);
+            }
+            assert(res.second);
+            return std::make_optional(res.second);
+        }
+    }
+
+    return std::nullopt;
+}
+
+
 void Store::queryPathInfo(const StorePath & storePath,
     Callback<ref<const ValidPathInfo>> callback) noexcept
 {
     auto hashPart = std::string(storePath.hashPart());
 
     try {
-        {
-            auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
-            if (res && res->isKnownNow()) {
-                stats.narInfoReadAverted++;
-                if (!res->didExist())
-                    throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
-                return callback(ref<const ValidPathInfo>(res->value));
-            }
+        auto r = queryPathInfoFromClientCache(storePath);
+        if (r.has_value()) {
+            std::shared_ptr<const ValidPathInfo> & info = *r;
+            if (info)
+                return callback(ref(info));
+            else
+                throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
         }
-
-        if (diskCache) {
-            auto res = diskCache->lookupNarInfo(getUri(), hashPart);
-            if (res.first != NarInfoDiskCache::oUnknown) {
-                stats.narInfoReadAverted++;
-                {
-                    auto state_(state.lock());
-                    state_->pathInfoCache.upsert(std::string(storePath.to_string()),
-                        res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
-                    if (res.first == NarInfoDiskCache::oInvalid ||
-                        !goodStorePath(storePath, res.second->path))
-                        throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
-                }
-                return callback(ref<const ValidPathInfo>(res.second));
-            }
-        }
-
     } catch (...) { return callback.rethrow(); }
 
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
@@ -984,6 +999,11 @@ void copyStorePath(
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
+    /* Bail out early (before starting a download from srcStore) if
+       dstStore already has this path. */
+    if (!repair && dstStore.isValidPath(storePath))
+        return;
+
     auto srcUri = srcStore.getUri();
     auto dstUri = dstStore.getUri();
     auto storePathS = srcStore.printStorePath(storePath);

src/libstore/store-api.hh

@@ -13,6 +13,7 @@
 #include "path-info.hh"
 #include "repair-flag.hh"
 #include "store-dir-config.hh"
+#include "source-path.hh"
 
 #include <nlohmann/json_fwd.hpp>
 #include <atomic>
@@ -107,7 +108,7 @@ struct StoreConfig : public StoreDirConfig
 
     StoreConfig() = delete;
 
-    StringSet getDefaultSystemFeatures();
+    static StringSet getDefaultSystemFeatures();
 
     virtual ~StoreConfig() { }
 
@@ -281,6 +282,16 @@ public:
     void queryPathInfo(const StorePath & path,
         Callback<ref<const ValidPathInfo>> callback) noexcept;
 
+    /**
+     * Version of queryPathInfo() that only queries the local narinfo cache and not
+     * the actual store.
+     *
+     * @return `std::nullopt` if nothing is known about the path in the local narinfo cache.
+     * @return `std::make_optional(nullptr)` if the path is known to not exist.
+     * @return `std::make_optional(validPathInfo)` if the path is known to exist.
+     */
+    std::optional<std::shared_ptr<const ValidPathInfo>> queryPathInfoFromClientCache(const StorePath & path);
+
     /**
      * Query the information about a realisation.
      */
@@ -427,22 +438,28 @@ public:
      * libutil/archive.hh).
      */
     virtual StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method = FileIngestionMethod::Recursive,
-            HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
-            PathFilter & filter = defaultPathFilter,
-            RepairFlag repair = NoRepair,
-            const StorePathSet & references = StorePathSet());
+        std::string_view name,
+        SourceAccessor & accessor,
+        const CanonPath & path,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        PathFilter & filter = defaultPathFilter,
+        RepairFlag repair = NoRepair);
 
     /**
      * Copy the contents of a path to the store and register the
      * validity the resulting path, using a constant amount of
      * memory.
      */
-    ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
-                                 FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
-                                 std::optional<Hash> expectedCAHash = {});
+    ValidPathInfo addToStoreSlow(
+        std::string_view name,
+        SourceAccessor & accessor,
+        const CanonPath & path,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        std::optional<Hash> expectedCAHash = {});
 
     /**
      * Like addToStore(), but the contents of the path are contained
@@ -453,19 +470,12 @@ public:
      *
      * \todo remove?
      */
-    virtual StorePath addToStoreFromDump(Source & dump, std::string_view name,
-                                         FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair,
-                                         const StorePathSet & references = StorePathSet())
-    { unsupported("addToStoreFromDump"); }
-
-    /**
-     * Like addToStore, but the contents written to the output path is a
-     * regular file containing the given string.
-     */
-    virtual StorePath addTextToStore(
+    virtual StorePath addToStoreFromDump(
+        Source & dump,
         std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
         RepairFlag repair = NoRepair) = 0;
 
     /**
@@ -943,6 +953,7 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv);
 std::map<DrvOutput, StorePath> drvOutputReferences(
     Store & store,
     const Derivation & drv,
-    const StorePath & outputPath);
+    const StorePath & outputPath,
+    Store * evalStore = nullptr);
 
 }

src/libstore/store-dir-config.hh

@@ -86,41 +86,20 @@ struct StoreDirConfig : public Config
 
     StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
 
-    StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
-
     StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
 
     /**
-     * Read-only variant of addToStoreFromDump(). It returns the store
-     * path to which a NAR or flat file would be written.
+     * Read-only variant of addToStore(). It returns the store
+     * path for the given file sytem object.
      */
-    std::pair<StorePath, Hash> computeStorePathFromDump(
-        Source & dump,
+    std::pair<StorePath, Hash> computeStorePath(
         std::string_view name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        SourceAccessor & accessor,
+        const CanonPath & path,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
-        const StorePathSet & references = {}) const;
-
-    /**
-     * Preparatory part of addTextToStore().
-     *
-     * !!! Computation of the path should take the references given to
-     * addTextToStore() into account, otherwise we have a (relatively
-     * minor) security hole: a caller can register a source file with
-     * bogus references.  If there are too many references, the path may
-     * not be garbage collected when it has to be (not really a problem,
-     * the caller could create a root anyway), or it may be garbage
-     * collected when it shouldn't be (more serious).
-     *
-     * Hashing the references would solve this (bogus references would
-     * simply yield a different store path, so other users wouldn't be
-     * affected), but it has some backwards compatibility issues (the
-     * hashing scheme changes), so I'm not doing that for now.
-     */
-    StorePath computeStorePathForText(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references) const;
+        const StorePathSet & references = {},
+        PathFilter & filter = defaultPathFilter) const;
 };
 
 }

src/libstore/worker-protocol.cc

@@ -7,6 +7,7 @@
 #include "archive.hh"
 #include "path-info.hh"
 
+#include <chrono>
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -47,6 +48,31 @@ void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const StoreDirCon
 }
 
 
+std::optional<std::chrono::microseconds> WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
+{
+    auto tag = readNum<uint8_t>(conn.from);
+    switch (tag) {
+        case 0:
+            return std::nullopt;
+        case 1:
+            return std::optional<std::chrono::microseconds>{std::chrono::microseconds(readNum<int64_t>(conn.from))};
+        default:
+            throw Error("Invalid optional tag from remote");
+    }
+}
+
+void WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional<std::chrono::microseconds> & optDuration)
+{
+    if (!optDuration.has_value()) {
+        conn.to << uint8_t{0};
+    } else {
+        conn.to
+            << uint8_t{1}
+            << optDuration.value().count();
+    }
+}
+
+
 DerivedPath WorkerProto::Serialise<DerivedPath>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto s = readString(conn.from);
@@ -110,6 +136,10 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const StoreDirConfig & sto
             >> res.startTime
             >> res.stopTime;
     }
+    if (GET_PROTOCOL_MINOR(conn.version) >= 37) {
+        res.cpuUser = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
+        res.cpuSystem = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
+    }
     if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
         auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(store, conn);
         for (auto && [output, realisation] : builtOutputs)
@@ -132,6 +162,10 @@ void WorkerProto::Serialise<BuildResult>::write(const StoreDirConfig & store, Wo
             << res.startTime
             << res.stopTime;
     }
+    if (GET_PROTOCOL_MINOR(conn.version) >= 37) {
+        WorkerProto::write(store, conn, res.cpuUser);
+        WorkerProto::write(store, conn, res.cpuSystem);
+    }
     if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
         DrvOutputs builtOutputs;
         for (auto & [output, realisation] : res.builtOutputs)

src/libstore/worker-protocol.hh

@@ -1,6 +1,8 @@
 #pragma once
 ///@file
 
+#include <chrono>
+
 #include "common-protocol.hh"
 
 namespace nix {
@@ -9,7 +11,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION (1 << 8 | 36)
+#define PROTOCOL_VERSION (1 << 8 | 37)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -214,6 +216,8 @@ template<>
 DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo);
 template<>
 DECLARE_WORKER_SERIALISER(std::optional<TrustedFlag>);
+template<>
+DECLARE_WORKER_SERIALISER(std::optional<std::chrono::microseconds>);
 
 template<typename T>
 DECLARE_WORKER_SERIALISER(std::vector<T>);