Merge pull request #14273 from fzakaria/fzakaria/issue-13944

Make `nix nar [cat|ls]` lazy

src/libstore/include/nix/store/nar-accessor.hh

@@ -27,7 +27,12 @@ ref<SourceAccessor> makeNarAccessor(Source & source);
  */
 using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
 
-ref<SourceAccessor> makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes);
+/**
+ * The canonical GetNarBytes function for a seekable Source.
+ */
+GetNarBytes seekableGetNarBytes(const Path & path);
+
+ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
 
 /**
  * Write a JSON representation of the contents of a NAR (except file

src/libstore/nar-accessor.cc

@@ -141,14 +141,14 @@ struct NarAccessor : public SourceAccessor
         parseDump(indexer, indexer);
     }
 
-    NarAccessor(const std::string & listing, GetNarBytes getNarBytes)
+    NarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes)
         : getNarBytes(getNarBytes)
     {
         using json = nlohmann::json;
 
-        std::function<void(NarMember &, json &)> recurse;
+        std::function<void(NarMember &, const json &)> recurse;
 
-        recurse = [&](NarMember & member, json & v) {
+        recurse = [&](NarMember & member, const json & v) {
             std::string type = v["type"];
 
             if (type == "directory") {
@@ -169,8 +169,7 @@ struct NarAccessor : public SourceAccessor
                 return;
         };
 
-        json v = json::parse(listing);
+        recurse(root, listing);
-        recurse(root, v);
     }
 
     NarMember * find(const CanonPath & path)
@@ -251,11 +250,34 @@ ref<SourceAccessor> makeNarAccessor(Source & source)
     return make_ref<NarAccessor>(source);
 }
 
-ref<SourceAccessor> makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes)
+ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes)
 {
     return make_ref<NarAccessor>(listing, getNarBytes);
 }
 
+GetNarBytes seekableGetNarBytes(const Path & path)
+{
+    return [path](uint64_t offset, uint64_t length) {
+        AutoCloseFD fd = toDescriptor(open(
+            path.c_str(),
+            O_RDONLY
+#ifndef _WIN32
+                | O_CLOEXEC
+#endif
+            ));
+        if (!fd)
+            throw SysError("opening NAR cache file '%s'", path);
+
+        if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset)
+            throw SysError("seeking in '%s'", path);
+
+        std::string buf(length, 0);
+        readFull(fd.get(), buf.data(), length);
+
+        return buf;
+    };
+}
+
 using nlohmann::json;
 
 json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)

src/libstore/remote-fs-accessor.cc

@@ -70,26 +70,8 @@ std::shared_ptr<SourceAccessor> RemoteFSAccessor::accessObject(const StorePath &
 
         try {
             listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
-
+            auto listingJson = nlohmann::json::parse(listing);
-            auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) {
+            auto narAccessor = makeLazyNarAccessor(listingJson, seekableGetNarBytes(cacheFile));
-                AutoCloseFD fd = toDescriptor(open(
-                    cacheFile.c_str(),
-                    O_RDONLY
-#ifndef _WIN32
-                        | O_CLOEXEC
-#endif
-                    ));
-                if (!fd)
-                    throw SysError("opening NAR cache file '%s'", cacheFile);
-
-                if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset)
-                    throw SysError("seeking in '%s'", cacheFile);
-
-                std::string buf(length, 0);
-                readFull(fd.get(), buf.data(), length);
-
-                return buf;
-            });
 
             nars.emplace(storePath.hashPart(), narAccessor);
             return narAccessor;

src/nix/cat.cc

@@ -1,6 +1,10 @@
 #include "nix/cmd/command.hh"
 #include "nix/store/store-api.hh"
 #include "nix/store/nar-accessor.hh"
+#include "nix/util/serialise.hh"
+#include "nix/util/source-accessor.hh"
+
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 
@@ -71,7 +75,11 @@ struct CmdCatNar : StoreCommand, MixCat
 
     void run(ref<Store> store) override
     {
-        cat(makeNarAccessor(readFile(narPath)), CanonPath{path});
+        AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
+        auto source = FdSource{fd.get()};
+        auto narAccessor = makeNarAccessor(source);
+        auto listing = listNar(narAccessor, CanonPath::root, true);
+        cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };
 

src/nix/ls.cc

@@ -145,7 +145,11 @@ struct CmdLsNar : Command, MixLs
 
     void run() override
     {
-        list(makeNarAccessor(readFile(narPath)), CanonPath{path});
+        AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
+        auto source = FdSource{fd.get()};
+        auto narAccessor = makeNarAccessor(source);
+        auto listing = listNar(narAccessor, CanonPath::root, true);
+        list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };