Deduplicate listNar and MemorySourceAccessor::File

`listNar` did the not-so-pretty thing of going straight to JSON. Now it
uses `MemorySourceAccessor::File`, or rather variations of it, to go to
a C++ data type first, and only JSON second.

To accomplish this we add some type parameters to the `File` data type.
Actually, we need to do two rounds of this, because shallow NAR
listings. There is `FileT` and `DirectoryT` accordingly.

src/libstore/binary-cache-store.cc

@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     if (config.writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
-            {"root", listNar(*narAccessor, CanonPath::root, true)},
+            {"root", listNarDeep(*narAccessor, CanonPath::root)},
         };
 
         upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");

src/libstore/remote-fs-accessor.cc

@@ -39,7 +39,7 @@ ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std:
 
     if (cacheDir != "") {
         try {
-            nlohmann::json j = listNar(*narAccessor, CanonPath::root, true);
+            nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
             writeFile(makeCacheFile(hashPart, "ls"), j.dump());
         } catch (...) {
             ignoreExceptionExceptInterrupt();

src/libutil/include/nix/util/memory-source-accessor.hh

@@ -4,59 +4,111 @@
 #include "nix/util/source-path.hh"
 #include "nix/util/fs-sink.hh"
 #include "nix/util/variant-wrapper.hh"
+#include "nix/util/json-impls.hh"
 
 namespace nix {
 
+/**
+ * File System Object definitions
+ *
+ * @see https://nix.dev/manual/nix/latest/store/file-system-object.html
+ */
+namespace fso {
+
+template<typename RegularContents>
+struct Regular
+{
+    bool executable = false;
+    RegularContents contents;
+
+    auto operator<=>(const Regular &) const = default;
+};
+
+/**
+ * Child parameter because sometimes we want "shallow" directories without
+ * full file children.
+ */
+template<typename Child>
+struct DirectoryT
+{
+    using Name = std::string;
+
+    std::map<Name, Child, std::less<>> entries;
+
+    inline bool operator==(const DirectoryT &) const noexcept;
+    inline std::strong_ordering operator<=>(const DirectoryT &) const noexcept;
+};
+
+struct Symlink
+{
+    std::string target;
+
+    auto operator<=>(const Symlink &) const = default;
+};
+
+/**
+ * For when we know there is child, but don't know anything about it.
+ *
+ * This is not part of the core File System Object data model --- this
+ * represents not knowing, not an additional type of file.
+ */
+struct Opaque
+{
+    auto operator<=>(const Opaque &) const = default;
+};
+
+/**
+ * `File<std::string>` nicely defining what a "file system object"
+ * is in Nix.
+ *
+ * With a different type arugment, it is also can be a "skeletal"
+ * version is that abstract syntax for a "NAR listing".
+ */
+template<typename RegularContents, bool recur>
+struct VariantT
+{
+    bool operator==(const VariantT &) const noexcept;
+    std::strong_ordering operator<=>(const VariantT &) const noexcept;
+
+    using Regular = nix::fso::Regular<RegularContents>;
+
+    /**
+     * In the default case, we do want full file children for our directory.
+     */
+    using Directory = nix::fso::DirectoryT<std::conditional_t<recur, VariantT, Opaque>>;
+
+    using Symlink = nix::fso::Symlink;
+
+    using Raw = std::variant<Regular, Directory, Symlink>;
+    Raw raw;
+
+    MAKE_WRAPPER_CONSTRUCTOR(VariantT);
+
+    SourceAccessor::Stat lstat() const;
+};
+
+template<typename Child>
+inline bool DirectoryT<Child>::operator==(const DirectoryT &) const noexcept = default;
+
+template<typename Child>
+inline std::strong_ordering DirectoryT<Child>::operator<=>(const DirectoryT &) const noexcept = default;
+
+template<typename RegularContents, bool recur>
+inline bool
+VariantT<RegularContents, recur>::operator==(const VariantT<RegularContents, recur> &) const noexcept = default;
+
+template<typename RegularContents, bool recur>
+inline std::strong_ordering
+VariantT<RegularContents, recur>::operator<=>(const VariantT<RegularContents, recur> &) const noexcept = default;
+
+} // namespace fso
+
 /**
  * An source accessor for an in-memory file system.
  */
 struct MemorySourceAccessor : virtual SourceAccessor
 {
-    /**
+    using File = fso::VariantT<std::string, true>;
-     * In addition to being part of the implementation of
-     * `MemorySourceAccessor`, this has a side benefit of nicely
-     * defining what a "file system object" is in Nix.
-     */
-    struct File
-    {
-        bool operator==(const File &) const noexcept;
-        std::strong_ordering operator<=>(const File &) const noexcept;
-
-        struct Regular
-        {
-            bool executable = false;
-            std::string contents;
-
-            bool operator==(const Regular &) const = default;
-            auto operator<=>(const Regular &) const = default;
-        };
-
-        struct Directory
-        {
-            using Name = std::string;
-
-            std::map<Name, File, std::less<>> entries;
-
-            bool operator==(const Directory &) const noexcept;
-            // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
-            bool operator<(const Directory &) const noexcept;
-        };
-
-        struct Symlink
-        {
-            std::string target;
-
-            bool operator==(const Symlink &) const = default;
-            auto operator<=>(const Symlink &) const = default;
-        };
-
-        using Raw = std::variant<Regular, Directory, Symlink>;
-        Raw raw;
-
-        MAKE_WRAPPER_CONSTRUCTOR(File);
-
-        Stat lstat() const;
-    };
 
     std::optional<File> root;
 
@@ -89,19 +141,6 @@ struct MemorySourceAccessor : virtual SourceAccessor
     SourcePath addFile(CanonPath path, std::string && contents);
 };
 
-inline bool MemorySourceAccessor::File::Directory::operator==(
-    const MemorySourceAccessor::File::Directory &) const noexcept = default;
-
-inline bool
-MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept
-{
-    return entries < other.entries;
-}
-
-inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default;
-inline std::strong_ordering
-MemorySourceAccessor::File::operator<=>(const MemorySourceAccessor::File &) const noexcept = default;
-
 /**
  * Write to a `MemorySourceAccessor` at the given path
  */

src/libutil/include/nix/util/nar-accessor.hh

@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "nix/util/source-accessor.hh"
+#include "nix/util/memory-source-accessor.hh"
 
 #include <functional>
 
@@ -34,10 +34,55 @@ GetNarBytes seekableGetNarBytes(const Path & path);
 
 ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
 
+struct NarListingRegularFile
+{
+    /**
+     * @see `SourceAccessor::Stat::fileSize`
+     */
+    std::optional<uint64_t> fileSize;
+
+    /**
+     * @see `SourceAccessor::Stat::narOffset`
+     *
+     * We only set to non-`std::nullopt` if it is also non-zero.
+     */
+    std::optional<uint64_t> narOffset;
+
+    auto operator<=>(const NarListingRegularFile &) const = default;
+};
+
 /**
- * Write a JSON representation of the contents of a NAR (except file
+ * Abstract syntax for a "NAR listing".
- * contents).
  */
-nlohmann::json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse);
+using NarListing = fso::VariantT<NarListingRegularFile, true>;
+
+/**
+ * Shallow NAR listing where directory children are not recursively expanded.
+ * Uses a variant that can hold Regular/Symlink fully, but Directory children
+ * are just unit types indicating presence without content.
+ */
+using ShallowNarListing = fso::VariantT<NarListingRegularFile, false>;
+
+/**
+ * Return a deep structured representation of the contents of a NAR (except file
+ * contents), recursively listing all children.
+ */
+NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path);
+
+/**
+ * Return a shallow structured representation of the contents of a NAR (except file
+ * contents), only listing immediate children without recursing.
+ */
+ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path);
+
+/**
+ * Serialize a NarListing to JSON.
+ */
+void to_json(nlohmann::json & j, const NarListing & listing);
+
+/**
+ * Serialize a ShallowNarListing to JSON.
+ */
+void to_json(nlohmann::json & j, const ShallowNarListing & listing);
 
 } // namespace nix

src/libutil/memory-source-accessor.cc

@@ -68,25 +68,26 @@ bool MemorySourceAccessor::pathExists(const CanonPath & path)
     return open(path, std::nullopt);
 }
 
-MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const
+template<>
+SourceAccessor::Stat MemorySourceAccessor::File::lstat() const
 {
     return std::visit(
         overloaded{
             [](const Regular & r) {
-                return Stat{
+                return SourceAccessor::Stat{
-                    .type = tRegular,
+                    .type = SourceAccessor::tRegular,
                     .fileSize = r.contents.size(),
                     .isExecutable = r.executable,
                 };
             },
             [](const Directory &) {
-                return Stat{
+                return SourceAccessor::Stat{
-                    .type = tDirectory,
+                    .type = SourceAccessor::tDirectory,
                 };
             },
             [](const Symlink &) {
-                return Stat{
+                return SourceAccessor::Stat{
-                    .type = tSymlink,
+                    .type = SourceAccessor::tSymlink,
                 };
             },
         },

src/libutil/nar-accessor.cc

@@ -272,41 +272,39 @@ GetNarBytes seekableGetNarBytes(const Path & path)
     };
 }
 
-using nlohmann::json;
+template<bool deep>
+using ListNarResult = std::conditional_t<deep, NarListing, ShallowNarListing>;
 
-json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse)
+template<bool deep>
+static ListNarResult<deep> listNarImpl(SourceAccessor & accessor, const CanonPath & path)
 {
     auto st = accessor.lstat(path);
 
-    json obj = json::object();
-
     switch (st.type) {
     case SourceAccessor::Type::tRegular:
-        obj["type"] = "regular";
+        return typename ListNarResult<deep>::Regular{
-        if (st.fileSize)
+            .executable = st.isExecutable,
-            obj["size"] = *st.fileSize;
+            .contents =
-        if (st.isExecutable)
+                NarListingRegularFile{
-            obj["executable"] = true;
+                    .fileSize = st.fileSize,
-        if (st.narOffset && *st.narOffset)
+                    .narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt,
-            obj["narOffset"] = *st.narOffset;
+                },
-        break;
+        };
-    case SourceAccessor::Type::tDirectory:
+    case SourceAccessor::Type::tDirectory: {
-        obj["type"] = "directory";
+        typename ListNarResult<deep>::Directory dir;
-        {
+        for (const auto & [name, type] : accessor.readDirectory(path)) {
-            obj["entries"] = json::object();
+            if constexpr (deep) {
-            json & res2 = obj["entries"];
+                dir.entries.emplace(name, listNarImpl<true>(accessor, path / name));
-            for (const auto & [name, type] : accessor.readDirectory(path)) {
+            } else {
-                if (recurse) {
+                dir.entries.emplace(name, fso::Opaque{});
-                    res2[name] = listNar(accessor, path / name, true);
-                } else
-                    res2[name] = json::object();
             }
         }
-        break;
+        return dir;
+    }
     case SourceAccessor::Type::tSymlink:
-        obj["type"] = "symlink";
+        return typename ListNarResult<deep>::Symlink{
-        obj["target"] = accessor.readLink(path);
+            .target = accessor.readLink(path),
-        break;
+        };
     case SourceAccessor::Type::tBlock:
     case SourceAccessor::Type::tChar:
     case SourceAccessor::Type::tSocket:
@@ -314,7 +312,64 @@ json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse)
     case SourceAccessor::Type::tUnknown:
         assert(false); // cannot happen for NARs
     }
-    return obj;
+}
+
+NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path)
+{
+    return listNarImpl<true>(accessor, path);
+}
+
+ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path)
+{
+    return listNarImpl<false>(accessor, path);
+}
+
+template<typename Listing>
+static void to_json_impl(nlohmann::json & j, const Listing & listing)
+{
+    std::visit(
+        overloaded{
+            [&](const typename Listing::Regular & r) {
+                j = nlohmann::json::object();
+                j["type"] = "regular";
+                if (r.contents.fileSize)
+                    j["size"] = *r.contents.fileSize;
+                if (r.executable)
+                    j["executable"] = true;
+                if (r.contents.narOffset)
+                    j["narOffset"] = *r.contents.narOffset;
+            },
+            [&](const typename Listing::Directory & d) {
+                j = nlohmann::json::object();
+                j["type"] = "directory";
+                j["entries"] = nlohmann::json::object();
+                for (const auto & [name, child] : d.entries) {
+                    if constexpr (std::is_same_v<Listing, NarListing>) {
+                        to_json(j["entries"][name], child);
+                    } else if constexpr (std::is_same_v<Listing, ShallowNarListing>) {
+                        j["entries"][name] = nlohmann::json::object();
+                    } else {
+                        static_assert(false);
+                    }
+                }
+            },
+            [&](const typename Listing::Symlink & s) {
+                j = nlohmann::json::object();
+                j["type"] = "symlink";
+                j["target"] = s.target;
+            },
+        },
+        listing.raw);
+}
+
+void to_json(nlohmann::json & j, const NarListing & listing)
+{
+    to_json_impl(j, listing);
+}
+
+void to_json(nlohmann::json & j, const ShallowNarListing & listing)
+{
+    to_json_impl(j, listing);
 }
 
 } // namespace nix

src/nix/cat.cc

@@ -80,7 +80,7 @@ struct CmdCatNar : StoreCommand, MixCat
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};
         auto narAccessor = makeNarAccessor(source);
-        auto listing = listNar(*narAccessor, CanonPath::root, true);
+        nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root);
         cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };

src/nix/ls.cc

@@ -85,7 +85,12 @@ struct MixLs : virtual Args, MixJSON
         if (json) {
             if (showDirectory)
                 throw UsageError("'--directory' is useless with '--json'");
-            logger->cout("%s", listNar(*accessor, path, recursive));
+            nlohmann::json j;
+            if (recursive)
+                j = listNarDeep(*accessor, path);
+            else
+                j = listNarShallow(*accessor, path);
+            logger->cout("%s", j.dump());
         } else
             listText(accessor, std::move(path));
     }
@@ -150,7 +155,7 @@ struct CmdLsNar : Command, MixLs
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};
         auto narAccessor = makeNarAccessor(source);
-        auto listing = listNar(*narAccessor, CanonPath::root, true);
+        nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root);
         list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };