Merge pull request #14598 from NixOS/nar-listing-dedup

Deduplicate `listNar` and `MemorySourceAccessor::File`

src/libstore-tests/references.cc

@@ -99,7 +99,7 @@ TEST(references, scanForReferencesDeep)
     // Create an in-memory file system with various reference patterns
     auto accessor = make_ref<MemorySourceAccessor>();
     accessor->root = File::Directory{
-        .contents{
+        .entries{
             {
                 // file1.txt: contains hash1
                 "file1.txt",
@@ -125,7 +125,7 @@ TEST(references, scanForReferencesDeep)
                 // subdir: a subdirectory
                 "subdir",
                 File::Directory{
-                    .contents{
+                    .entries{
                         {
                             // subdir/file4.txt: contains hash1 again
                             "file4.txt",

src/libstore/binary-cache-store.cc

@@ -8,7 +8,7 @@
 #include "nix/util/sync.hh"
 #include "nix/store/remote-fs-accessor.hh"
 #include "nix/store/nar-info-disk-cache.hh"
-#include "nix/store/nar-accessor.hh"
+#include "nix/util/nar-accessor.hh"
 #include "nix/util/thread-pool.hh"
 #include "nix/util/callback.hh"
 #include "nix/util/signals.hh"
@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     if (config.writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
-            {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
+            {"root", listNarDeep(*narAccessor, CanonPath::root)},
         };
 
         upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");

src/libstore/include/nix/store/meson.build

@@ -55,7 +55,6 @@ headers = [ config_pub_h ] + files(
   'machines.hh',
   'make-content-addressed.hh',
   'names.hh',
-  'nar-accessor.hh',
   'nar-info-disk-cache.hh',
   'nar-info.hh',
   'outputs-spec.hh',

src/libstore/include/nix/store/nar-accessor.hh

@@ -1,43 +0,0 @@
-#pragma once
-///@file
-
-#include "nix/util/source-accessor.hh"
-
-#include <functional>
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix {
-
-struct Source;
-
-/**
- * Return an object that provides access to the contents of a NAR
- * file.
- */
-ref<SourceAccessor> makeNarAccessor(std::string && nar);
-
-ref<SourceAccessor> makeNarAccessor(Source & source);
-
-/**
- * Create a NAR accessor from a NAR listing (in the format produced by
- * listNar()). The callback getNarBytes(offset, length) is used by the
- * readFile() method of the accessor to get the contents of files
- * inside the NAR.
- */
-using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
-
-/**
- * The canonical GetNarBytes function for a seekable Source.
- */
-GetNarBytes seekableGetNarBytes(const Path & path);
-
-ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
-
-/**
- * Write a JSON representation of the contents of a NAR (except file
- * contents).
- */
-nlohmann::json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse);
-
-} // namespace nix

src/libstore/meson.build

@@ -300,7 +300,6 @@ sources = files(
   'make-content-addressed.cc',
   'misc.cc',
   'names.cc',
-  'nar-accessor.cc',
   'nar-info-disk-cache.cc',
   'nar-info.cc',
   'optimise-store.cc',

src/libstore/remote-fs-accessor.cc

@@ -1,6 +1,6 @@
 #include <nlohmann/json.hpp>
 #include "nix/store/remote-fs-accessor.hh"
-#include "nix/store/nar-accessor.hh"
+#include "nix/util/nar-accessor.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -39,7 +39,7 @@ ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std:
 
     if (cacheDir != "") {
         try {
-            nlohmann::json j = listNar(narAccessor, CanonPath::root, true);
+            nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
             writeFile(makeCacheFile(hashPart, "ls"), j.dump());
         } catch (...) {
             ignoreExceptionExceptInterrupt();

src/libutil-tests/git.cc

@@ -230,7 +230,7 @@ TEST_F(GitTest, both_roundrip)
 
     auto files = make_ref<MemorySourceAccessor>();
     files->root = File::Directory{
-        .contents{
+        .entries{
             {
                 "foo",
                 File::Regular{
@@ -240,7 +240,7 @@ TEST_F(GitTest, both_roundrip)
             {
                 "bar",
                 File::Directory{
-                    .contents =
+                    .entries =
                         {
                             {
                                 "baz",

src/libutil/include/nix/util/memory-source-accessor.hh

@@ -4,60 +4,112 @@
 #include "nix/util/source-path.hh"
 #include "nix/util/fs-sink.hh"
 #include "nix/util/variant-wrapper.hh"
+#include "nix/util/json-impls.hh"
 
 namespace nix {
 
 /**
- * An source accessor for an in-memory file system.
+ * File System Object definitions
+ *
+ * @see https://nix.dev/manual/nix/latest/store/file-system-object.html
  */
-struct MemorySourceAccessor : virtual SourceAccessor
-{
-    /**
-     * In addition to being part of the implementation of
-     * `MemorySourceAccessor`, this has a side benefit of nicely
-     * defining what a "file system object" is in Nix.
-     */
-    struct File
-    {
-        bool operator==(const File &) const noexcept;
-        std::strong_ordering operator<=>(const File &) const noexcept;
+namespace fso {
 
+template<typename RegularContents>
 struct Regular
 {
     bool executable = false;
-            std::string contents;
+    RegularContents contents;
 
-            bool operator==(const Regular &) const = default;
     auto operator<=>(const Regular &) const = default;
 };
 
-        struct Directory
+/**
+ * Child parameter because sometimes we want "shallow" directories without
+ * full file children.
+ */
+template<typename Child>
+struct DirectoryT
 {
     using Name = std::string;
 
-            std::map<Name, File, std::less<>> contents;
+    std::map<Name, Child, std::less<>> entries;
 
-            bool operator==(const Directory &) const noexcept;
-            // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
-            bool operator<(const Directory &) const noexcept;
+    inline bool operator==(const DirectoryT &) const noexcept;
+    inline std::strong_ordering operator<=>(const DirectoryT &) const noexcept;
 };
 
 struct Symlink
 {
     std::string target;
 
-            bool operator==(const Symlink &) const = default;
     auto operator<=>(const Symlink &) const = default;
 };
 
+/**
+ * For when we know there is child, but don't know anything about it.
+ *
+ * This is not part of the core File System Object data model --- this
+ * represents not knowing, not an additional type of file.
+ */
+struct Opaque
+{
+    auto operator<=>(const Opaque &) const = default;
+};
+
+/**
+ * `File<std::string>` nicely defining what a "file system object"
+ * is in Nix.
+ *
+ * With a different type arugment, it is also can be a "skeletal"
+ * version is that abstract syntax for a "NAR listing".
+ */
+template<typename RegularContents, bool recur>
+struct VariantT
+{
+    bool operator==(const VariantT &) const noexcept;
+    std::strong_ordering operator<=>(const VariantT &) const noexcept;
+
+    using Regular = nix::fso::Regular<RegularContents>;
+
+    /**
+     * In the default case, we do want full file children for our directory.
+     */
+    using Directory = nix::fso::DirectoryT<std::conditional_t<recur, VariantT, Opaque>>;
+
+    using Symlink = nix::fso::Symlink;
+
     using Raw = std::variant<Regular, Directory, Symlink>;
     Raw raw;
 
-        MAKE_WRAPPER_CONSTRUCTOR(File);
+    MAKE_WRAPPER_CONSTRUCTOR(VariantT);
 
-        Stat lstat() const;
+    SourceAccessor::Stat lstat() const;
 };
 
+template<typename Child>
+inline bool DirectoryT<Child>::operator==(const DirectoryT &) const noexcept = default;
+
+template<typename Child>
+inline std::strong_ordering DirectoryT<Child>::operator<=>(const DirectoryT &) const noexcept = default;
+
+template<typename RegularContents, bool recur>
+inline bool
+VariantT<RegularContents, recur>::operator==(const VariantT<RegularContents, recur> &) const noexcept = default;
+
+template<typename RegularContents, bool recur>
+inline std::strong_ordering
+VariantT<RegularContents, recur>::operator<=>(const VariantT<RegularContents, recur> &) const noexcept = default;
+
+} // namespace fso
+
+/**
+ * An source accessor for an in-memory file system.
+ */
+struct MemorySourceAccessor : virtual SourceAccessor
+{
+    using File = fso::VariantT<std::string, true>;
+
     std::optional<File> root;
 
     bool operator==(const MemorySourceAccessor &) const noexcept = default;
@@ -89,19 +141,6 @@ struct MemorySourceAccessor : virtual SourceAccessor
     SourcePath addFile(CanonPath path, std::string && contents);
 };
 
-inline bool MemorySourceAccessor::File::Directory::operator==(
-    const MemorySourceAccessor::File::Directory &) const noexcept = default;
-
-inline bool
-MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept
-{
-    return contents < other.contents;
-}
-
-inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default;
-inline std::strong_ordering
-MemorySourceAccessor::File::operator<=>(const MemorySourceAccessor::File &) const noexcept = default;
-
 /**
  * Write to a `MemorySourceAccessor` at the given path
  */

src/libutil/include/nix/util/meson.build

@@ -50,6 +50,7 @@ headers = files(
   'memory-source-accessor.hh',
   'mounted-source-accessor.hh',
   'muxable-pipe.hh',
+  'nar-accessor.hh',
   'os-string.hh',
   'pool.hh',
   'pos-idx.hh',

src/libutil/include/nix/util/nar-accessor.hh

@@ -0,0 +1,88 @@
+#pragma once
+///@file
+
+#include "nix/util/memory-source-accessor.hh"
+
+#include <functional>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+
+struct Source;
+
+/**
+ * Return an object that provides access to the contents of a NAR
+ * file.
+ */
+ref<SourceAccessor> makeNarAccessor(std::string && nar);
+
+ref<SourceAccessor> makeNarAccessor(Source & source);
+
+/**
+ * Create a NAR accessor from a NAR listing (in the format produced by
+ * listNar()). The callback getNarBytes(offset, length) is used by the
+ * readFile() method of the accessor to get the contents of files
+ * inside the NAR.
+ */
+using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
+
+/**
+ * The canonical GetNarBytes function for a seekable Source.
+ */
+GetNarBytes seekableGetNarBytes(const Path & path);
+
+ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
+
+struct NarListingRegularFile
+{
+    /**
+     * @see `SourceAccessor::Stat::fileSize`
+     */
+    std::optional<uint64_t> fileSize;
+
+    /**
+     * @see `SourceAccessor::Stat::narOffset`
+     *
+     * We only set to non-`std::nullopt` if it is also non-zero.
+     */
+    std::optional<uint64_t> narOffset;
+
+    auto operator<=>(const NarListingRegularFile &) const = default;
+};
+
+/**
+ * Abstract syntax for a "NAR listing".
+ */
+using NarListing = fso::VariantT<NarListingRegularFile, true>;
+
+/**
+ * Shallow NAR listing where directory children are not recursively expanded.
+ * Uses a variant that can hold Regular/Symlink fully, but Directory children
+ * are just unit types indicating presence without content.
+ */
+using ShallowNarListing = fso::VariantT<NarListingRegularFile, false>;
+
+/**
+ * Return a deep structured representation of the contents of a NAR (except file
+ * contents), recursively listing all children.
+ */
+NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path);
+
+/**
+ * Return a shallow structured representation of the contents of a NAR (except file
+ * contents), only listing immediate children without recursing.
+ */
+ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path);
+
+/**
+ * Serialize a NarListing to JSON.
+ */
+void to_json(nlohmann::json & j, const NarListing & listing);
+
+/**
+ * Serialize a ShallowNarListing to JSON.
+ */
+void to_json(nlohmann::json & j, const ShallowNarListing & listing);
+
+} // namespace nix

src/libutil/memory-source-accessor.cc

@@ -29,13 +29,13 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path,
             return nullptr;
         auto & curDir = *curDirP;
 
-        auto i = curDir.contents.find(name);
-        if (i == curDir.contents.end()) {
+        auto i = curDir.entries.find(name);
+        if (i == curDir.entries.end()) {
             if (!create)
                 return nullptr;
             else {
                 newF = true;
-                i = curDir.contents.insert(
+                i = curDir.entries.insert(
                     i,
                     {
                         std::string{name},
@@ -68,25 +68,26 @@ bool MemorySourceAccessor::pathExists(const CanonPath & path)
     return open(path, std::nullopt);
 }
 
-MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const
+template<>
+SourceAccessor::Stat MemorySourceAccessor::File::lstat() const
 {
     return std::visit(
         overloaded{
             [](const Regular & r) {
-                return Stat{
-                    .type = tRegular,
+                return SourceAccessor::Stat{
+                    .type = SourceAccessor::tRegular,
                     .fileSize = r.contents.size(),
                     .isExecutable = r.executable,
                 };
             },
             [](const Directory &) {
-                return Stat{
-                    .type = tDirectory,
+                return SourceAccessor::Stat{
+                    .type = SourceAccessor::tDirectory,
                 };
             },
             [](const Symlink &) {
-                return Stat{
-                    .type = tSymlink,
+                return SourceAccessor::Stat{
+                    .type = SourceAccessor::tSymlink,
                 };
             },
         },
@@ -106,7 +107,7 @@ MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const Canon
         throw Error("file '%s' does not exist", path);
     if (auto * d = std::get_if<File::Directory>(&f->raw)) {
         DirEntries res;
-        for (auto & [name, file] : d->contents)
+        for (auto & [name, file] : d->entries)
             res.insert_or_assign(name, file.lstat().type);
         return res;
     } else

src/libutil/meson.build

@@ -147,6 +147,7 @@ sources = [ config_priv_h ] + files(
   'logging.cc',
   'memory-source-accessor.cc',
   'mounted-source-accessor.cc',
+  'nar-accessor.cc',
   'pos-table.cc',
   'position.cc',
   'posix-source-accessor.cc',

src/libutil/nar-accessor.cc

@@ -1,4 +1,4 @@
-#include "nix/store/nar-accessor.hh"
+#include "nix/util/nar-accessor.hh"
 #include "nix/util/archive.hh"
 
 #include <map>
@@ -272,41 +272,39 @@ GetNarBytes seekableGetNarBytes(const Path & path)
     };
 }
 
-using nlohmann::json;
+template<bool deep>
+using ListNarResult = std::conditional_t<deep, NarListing, ShallowNarListing>;
 
-json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
+template<bool deep>
+static ListNarResult<deep> listNarImpl(SourceAccessor & accessor, const CanonPath & path)
 {
-    auto st = accessor->lstat(path);
-
-    json obj = json::object();
+    auto st = accessor.lstat(path);
 
     switch (st.type) {
     case SourceAccessor::Type::tRegular:
-        obj["type"] = "regular";
-        if (st.fileSize)
-            obj["size"] = *st.fileSize;
-        if (st.isExecutable)
-            obj["executable"] = true;
-        if (st.narOffset && *st.narOffset)
-            obj["narOffset"] = *st.narOffset;
-        break;
-    case SourceAccessor::Type::tDirectory:
-        obj["type"] = "directory";
-        {
-            obj["entries"] = json::object();
-            json & res2 = obj["entries"];
-            for (const auto & [name, type] : accessor->readDirectory(path)) {
-                if (recurse) {
-                    res2[name] = listNar(accessor, path / name, true);
-                } else
-                    res2[name] = json::object();
+        return typename ListNarResult<deep>::Regular{
+            .executable = st.isExecutable,
+            .contents =
+                NarListingRegularFile{
+                    .fileSize = st.fileSize,
+                    .narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt,
+                },
+        };
+    case SourceAccessor::Type::tDirectory: {
+        typename ListNarResult<deep>::Directory dir;
+        for (const auto & [name, type] : accessor.readDirectory(path)) {
+            if constexpr (deep) {
+                dir.entries.emplace(name, listNarImpl<true>(accessor, path / name));
+            } else {
+                dir.entries.emplace(name, fso::Opaque{});
             }
         }
-        break;
+        return dir;
+    }
     case SourceAccessor::Type::tSymlink:
-        obj["type"] = "symlink";
-        obj["target"] = accessor->readLink(path);
-        break;
+        return typename ListNarResult<deep>::Symlink{
+            .target = accessor.readLink(path),
+        };
     case SourceAccessor::Type::tBlock:
     case SourceAccessor::Type::tChar:
     case SourceAccessor::Type::tSocket:
@@ -314,7 +312,64 @@ json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
     case SourceAccessor::Type::tUnknown:
         assert(false); // cannot happen for NARs
     }
-    return obj;
+}
+
+NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path)
+{
+    return listNarImpl<true>(accessor, path);
+}
+
+ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path)
+{
+    return listNarImpl<false>(accessor, path);
+}
+
+template<typename Listing>
+static void to_json_impl(nlohmann::json & j, const Listing & listing)
+{
+    std::visit(
+        overloaded{
+            [&](const typename Listing::Regular & r) {
+                j = nlohmann::json::object();
+                j["type"] = "regular";
+                if (r.contents.fileSize)
+                    j["size"] = *r.contents.fileSize;
+                if (r.executable)
+                    j["executable"] = true;
+                if (r.contents.narOffset)
+                    j["narOffset"] = *r.contents.narOffset;
+            },
+            [&](const typename Listing::Directory & d) {
+                j = nlohmann::json::object();
+                j["type"] = "directory";
+                j["entries"] = nlohmann::json::object();
+                for (const auto & [name, child] : d.entries) {
+                    if constexpr (std::is_same_v<Listing, NarListing>) {
+                        to_json(j["entries"][name], child);
+                    } else if constexpr (std::is_same_v<Listing, ShallowNarListing>) {
+                        j["entries"][name] = nlohmann::json::object();
+                    } else {
+                        static_assert(false);
+                    }
+                }
+            },
+            [&](const typename Listing::Symlink & s) {
+                j = nlohmann::json::object();
+                j["type"] = "symlink";
+                j["target"] = s.target;
+            },
+        },
+        listing.raw);
+}
+
+void to_json(nlohmann::json & j, const NarListing & listing)
+{
+    to_json_impl(j, listing);
+}
+
+void to_json(nlohmann::json & j, const ShallowNarListing & listing)
+{
+    to_json_impl(j, listing);
 }
 
 } // namespace nix

src/nix/cat.cc

@@ -1,6 +1,6 @@
 #include "nix/cmd/command.hh"
 #include "nix/store/store-api.hh"
-#include "nix/store/nar-accessor.hh"
+#include "nix/util/nar-accessor.hh"
 #include "nix/util/serialise.hh"
 #include "nix/util/source-accessor.hh"
 
@@ -80,7 +80,7 @@ struct CmdCatNar : StoreCommand, MixCat
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};
         auto narAccessor = makeNarAccessor(source);
-        auto listing = listNar(narAccessor, CanonPath::root, true);
+        nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root);
         cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };

src/nix/ls.cc

@@ -1,6 +1,6 @@
 #include "nix/cmd/command.hh"
 #include "nix/store/store-api.hh"
-#include "nix/store/nar-accessor.hh"
+#include "nix/util/nar-accessor.hh"
 #include "nix/main/common-args.hh"
 #include <nlohmann/json.hpp>
 
@@ -85,7 +85,12 @@ struct MixLs : virtual Args, MixJSON
         if (json) {
             if (showDirectory)
                 throw UsageError("'--directory' is useless with '--json'");
-            logger->cout("%s", listNar(accessor, path, recursive));
+            nlohmann::json j;
+            if (recursive)
+                j = listNarDeep(*accessor, path);
+            else
+                j = listNarShallow(*accessor, path);
+            logger->cout("%s", j.dump());
         } else
             listText(accessor, std::move(path));
     }
@@ -150,7 +155,7 @@ struct CmdLsNar : Command, MixLs
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};
         auto narAccessor = makeNarAccessor(source);
-        auto listing = listNar(narAccessor, CanonPath::root, true);
+        nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root);
         list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };