Merge remote-tracking branch 'upstream/master' into lfs

src/libstore/filetransfer.hh

@@ -50,8 +50,9 @@ struct FileTransferSettings : Config
 
     Setting<size_t> downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size",
         R"(
-          The size of Nix's internal download buffer during `curl` transfers. If data is
+          The size of Nix's internal download buffer in bytes during `curl` transfers. If data is
           not processed quickly enough to exceed the size of this buffer, downloads may stall.
+          The default is 67108864 (64 MiB).
         )"};
 };
 

src/libstore/gc.cc

@@ -455,7 +455,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     bool gcKeepOutputs = settings.gcKeepOutputs;
     bool gcKeepDerivations = settings.gcKeepDerivations;
 
-    StorePathSet roots, dead, alive;
+    std::unordered_set<StorePath> roots, dead, alive;
 
     struct Shared
     {
@@ -661,7 +661,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         }
     };
 
-    std::map<StorePath, StorePathSet> referrersCache;
+    std::unordered_map<StorePath, StorePathSet> referrersCache;
 
     /* Helper function that visits all paths reachable from `start`
        via the referrers edges and optionally derivers and derivation

src/libstore/nar-accessor.cc

@@ -291,7 +291,11 @@ json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
         obj["type"] = "symlink";
         obj["target"] = accessor->readLink(path);
         break;
-    case SourceAccessor::Type::tMisc:
+    case SourceAccessor::Type::tBlock:
+    case SourceAccessor::Type::tChar:
+    case SourceAccessor::Type::tSocket:
+    case SourceAccessor::Type::tFifo:
+    case SourceAccessor::Type::tUnknown:
         assert(false); // cannot happen for NARs
     }
     return obj;

src/libstore/package.nix

@@ -87,8 +87,6 @@ mkMesonLibrary (finalAttrs: {
     # https://github.com/NixOS/nixpkgs/issues/86131.
     BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
     BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
-  } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
-    LDFLAGS = "-fuse-ld=gold";
   };
 
   meta = {

src/libstore/remote-store.cc

@@ -534,14 +534,17 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 void RemoteStore::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
     auto source = sinkToSource([&](Sink & sink) {
         sink << pathsToCopy.size();
-        for (auto & [pathInfo, pathSource] : pathsToCopy) {
+        // Reverse, so we can release memory at the original start
+        std::reverse(pathsToCopy.begin(), pathsToCopy.end());
+        while (!pathsToCopy.empty()) {
+            auto & [pathInfo, pathSource] = pathsToCopy.back();
             WorkerProto::Serialise<ValidPathInfo>::write(*this,
                  WorkerProto::WriteConn {
                      .to = sink,
@@ -549,6 +552,7 @@ void RemoteStore::addMultipleToStore(
                  },
                  pathInfo);
             pathSource->drainInto(sink);
+            pathsToCopy.pop_back();
         }
     });
 

src/libstore/remote-store.hh

@@ -102,7 +102,7 @@ public:
         CheckSigsFlag checkSigs) override;
 
     void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair,
         CheckSigsFlag checkSigs) override;

src/libstore/store-api.cc

@@ -223,7 +223,7 @@ StorePath Store::addToStore(
 }
 
 void Store::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
@@ -246,9 +246,7 @@ void Store::addMultipleToStore(
         act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed);
     };
 
-    ThreadPool pool;
-
-    processGraph<StorePath>(pool,
+    processGraph<StorePath>(
         storePathsToAdd,
 
         [&](const StorePath & path) {
@@ -1028,12 +1026,10 @@ std::map<StorePath, StorePath> copyPaths(
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
 
-    ThreadPool pool;
-
     try {
         // Copy the realisation closure
         processGraph<Realisation>(
-            pool, Realisation::closure(srcStore, toplevelRealisations),
+            Realisation::closure(srcStore, toplevelRealisations),
             [&](const Realisation & current) -> std::set<Realisation> {
                 std::set<Realisation> children;
                 for (const auto & [drvOutput, _] : current.dependentRealisations) {
@@ -1142,7 +1138,7 @@ std::map<StorePath, StorePath> copyPaths(
         pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
     }
 
-    dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
+    dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);
 
     return pathsMap;
 }

src/libstore/store-api.hh

@@ -425,7 +425,7 @@ public:
         CheckSigsFlag checkSigs = CheckSigs);
 
     virtual void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair = NoRepair,
         CheckSigsFlag checkSigs = CheckSigs);