nix nar {ls,cat}: Optimize

The whole NarAccessor -> listing -> lazy NarAccessor is very weird. Source
can now be seek-ed over when supported, so we can support it pretty easily.
Alternatively we could also make it single-pass very easily with a custom
FileSystemObjectSink. It will get removed in a follow-up commit anyway.

src/libutil/include/nix/util/nar-accessor.hh

@@ -32,8 +32,16 @@ using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
  */
 GetNarBytes seekableGetNarBytes(const Path & path);
 
+GetNarBytes seekableGetNarBytes(Descriptor fd);
+
 ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
 
+/**
+ * Creates a NAR accessor from a given stream and a GetNarBytes getter.
+ * @param source Consumed eagerly. References to it are not persisted in the resulting SourceAccessor.
+ */
+ref<SourceAccessor> makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes);
+
 struct NarListingRegularFile
 {
     /**

src/libutil/nar-accessor.cc

@@ -111,6 +111,7 @@ struct NarAccessor : public SourceAccessor
                 path,
                 NarMember{.stat = {.type = Type::tRegular, .fileSize = 0, .isExecutable = false, .narOffset = 0}});
             NarMemberConstructor nmc{nm, pos};
+            nmc.skipContents = true; /* Don't care about contents. */
             func(nmc);
         }
 
@@ -141,6 +142,13 @@ struct NarAccessor : public SourceAccessor
         parseDump(indexer, indexer);
     }
 
+    NarAccessor(Source & source, GetNarBytes getNarBytes)
+        : getNarBytes(std::move(getNarBytes))
+    {
+        NarIndexer indexer(*this, source);
+        parseDump(indexer, indexer);
+    }
+
     NarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes)
         : getNarBytes(getNarBytes)
     {
@@ -249,24 +257,35 @@ ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBy
     return make_ref<NarAccessor>(listing, getNarBytes);
 }
 
+ref<SourceAccessor> makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes)
+{
+    return make_ref<NarAccessor>(source, getNarBytes);
+}
+
 GetNarBytes seekableGetNarBytes(const Path & path)
 {
-    return [path](uint64_t offset, uint64_t length) {
+    AutoCloseFD fd = toDescriptor(open(
-        AutoCloseFD fd = toDescriptor(open(
+        path.c_str(),
-            path.c_str(),
+        O_RDONLY
-            O_RDONLY
+#ifdef O_CLOEXEC
-#ifndef _WIN32
+            | O_CLOEXEC
-                | O_CLOEXEC
 #endif
-            ));
+        ));
-        if (!fd)
+    if (!fd)
-            throw SysError("opening NAR cache file '%s'", path);
+        throw SysError("opening NAR cache file '%s'", path);
 
-        if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset)
+    return [inner = seekableGetNarBytes(fd.get()), fd = make_ref<AutoCloseFD>(std::move(fd))](
-            throw SysError("seeking in '%s'", path);
+               uint64_t offset, uint64_t length) { return inner(offset, length); };
+}
+
+GetNarBytes seekableGetNarBytes(Descriptor fd)
+{
+    return [fd](uint64_t offset, uint64_t length) {
+        if (::lseek(fromDescriptorReadOnly(fd), offset, SEEK_SET) == -1)
+            throw SysError("seeking in file");
 
         std::string buf(length, 0);
-        readFull(fd.get(), buf.data(), length);
+        readFull(fd, buf.data(), length);
 
         return buf;
     };

src/nix/cat.cc

@@ -79,9 +79,7 @@ struct CmdCatNar : StoreCommand, MixCat
         if (!fd)
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};
-        auto narAccessor = makeNarAccessor(source);
+        cat(makeLazyNarAccessor(source, seekableGetNarBytes(fd.get())), CanonPath{path});
-        nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root);
-        cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };
 

src/nix/ls.cc

@@ -154,9 +154,7 @@ struct CmdLsNar : Command, MixLs
         if (!fd)
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};
-        auto narAccessor = makeNarAccessor(source);
+        list(makeLazyNarAccessor(source, seekableGetNarBytes(fd.get())), CanonPath{path});
-        nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root);
-        list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };