diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 49077524f..caae72479 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(*narAccessor, CanonPath::root, true)}, + {"root", listNarDeep(*narAccessor, CanonPath::root)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index e2c8f65a7..51bab9953 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -39,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std: if (cacheDir != "") { try { - nlohmann::json j = listNar(*narAccessor, CanonPath::root, true); + nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreExceptionExceptInterrupt(); diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index 6aadc53a3..268f6b06f 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -4,59 +4,111 @@ #include "nix/util/source-path.hh" #include "nix/util/fs-sink.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { +/** + * File System Object definitions + * + * @see https://nix.dev/manual/nix/latest/store/file-system-object.html + */ +namespace fso { + +template +struct Regular +{ + bool executable = false; + RegularContents contents; + + auto operator<=>(const Regular &) const = default; +}; + +/** + * Child parameter because sometimes we want "shallow" directories without + * full file children. + */ +template +struct DirectoryT +{ + using Name = std::string; + + std::map> entries; + + inline bool operator==(const DirectoryT &) const noexcept; + inline std::strong_ordering operator<=>(const DirectoryT &) const noexcept; +}; + +struct Symlink +{ + std::string target; + + auto operator<=>(const Symlink &) const = default; +}; + +/** + * For when we know there is child, but don't know anything about it. + * + * This is not part of the core File System Object data model --- this + * represents not knowing, not an additional type of file. + */ +struct Opaque +{ + auto operator<=>(const Opaque &) const = default; +}; + +/** + * `File` nicely defining what a "file system object" + * is in Nix. + * + * With a different type arugment, it is also can be a "skeletal" + * version is that abstract syntax for a "NAR listing". + */ +template +struct VariantT +{ + bool operator==(const VariantT &) const noexcept; + std::strong_ordering operator<=>(const VariantT &) const noexcept; + + using Regular = nix::fso::Regular; + + /** + * In the default case, we do want full file children for our directory. + */ + using Directory = nix::fso::DirectoryT>; + + using Symlink = nix::fso::Symlink; + + using Raw = std::variant; + Raw raw; + + MAKE_WRAPPER_CONSTRUCTOR(VariantT); + + SourceAccessor::Stat lstat() const; +}; + +template +inline bool DirectoryT::operator==(const DirectoryT &) const noexcept = default; + +template +inline std::strong_ordering DirectoryT::operator<=>(const DirectoryT &) const noexcept = default; + +template +inline bool +VariantT::operator==(const VariantT &) const noexcept = default; + +template +inline std::strong_ordering +VariantT::operator<=>(const VariantT &) const noexcept = default; + +} // namespace fso + /** * An source accessor for an in-memory file system. */ struct MemorySourceAccessor : virtual SourceAccessor { - /** - * In addition to being part of the implementation of - * `MemorySourceAccessor`, this has a side benefit of nicely - * defining what a "file system object" is in Nix. - */ - struct File - { - bool operator==(const File &) const noexcept; - std::strong_ordering operator<=>(const File &) const noexcept; - - struct Regular - { - bool executable = false; - std::string contents; - - bool operator==(const Regular &) const = default; - auto operator<=>(const Regular &) const = default; - }; - - struct Directory - { - using Name = std::string; - - std::map> entries; - - bool operator==(const Directory &) const noexcept; - // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. - bool operator<(const Directory &) const noexcept; - }; - - struct Symlink - { - std::string target; - - bool operator==(const Symlink &) const = default; - auto operator<=>(const Symlink &) const = default; - }; - - using Raw = std::variant; - Raw raw; - - MAKE_WRAPPER_CONSTRUCTOR(File); - - Stat lstat() const; - }; + using File = fso::VariantT; std::optional root; @@ -89,19 +141,6 @@ struct MemorySourceAccessor : virtual SourceAccessor SourcePath addFile(CanonPath path, std::string && contents); }; -inline bool MemorySourceAccessor::File::Directory::operator==( - const MemorySourceAccessor::File::Directory &) const noexcept = default; - -inline bool -MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept -{ - return entries < other.entries; -} - -inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default; -inline std::strong_ordering -MemorySourceAccessor::File::operator<=>(const MemorySourceAccessor::File &) const noexcept = default; - /** * Write to a `MemorySourceAccessor` at the given path */ diff --git a/src/libutil/include/nix/util/nar-accessor.hh b/src/libutil/include/nix/util/nar-accessor.hh index df7b0fcf2..9665af9bc 100644 --- a/src/libutil/include/nix/util/nar-accessor.hh +++ b/src/libutil/include/nix/util/nar-accessor.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/util/source-accessor.hh" +#include "nix/util/memory-source-accessor.hh" #include @@ -34,10 +34,55 @@ GetNarBytes seekableGetNarBytes(const Path & path); ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes); +struct NarListingRegularFile +{ + /** + * @see `SourceAccessor::Stat::fileSize` + */ + std::optional fileSize; + + /** + * @see `SourceAccessor::Stat::narOffset` + * + * We only set to non-`std::nullopt` if it is also non-zero. + */ + std::optional narOffset; + + auto operator<=>(const NarListingRegularFile &) const = default; +}; + /** - * Write a JSON representation of the contents of a NAR (except file - * contents). + * Abstract syntax for a "NAR listing". */ -nlohmann::json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse); +using NarListing = fso::VariantT; + +/** + * Shallow NAR listing where directory children are not recursively expanded. + * Uses a variant that can hold Regular/Symlink fully, but Directory children + * are just unit types indicating presence without content. + */ +using ShallowNarListing = fso::VariantT; + +/** + * Return a deep structured representation of the contents of a NAR (except file + * contents), recursively listing all children. + */ +NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path); + +/** + * Return a shallow structured representation of the contents of a NAR (except file + * contents), only listing immediate children without recursing. + */ +ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path); + +/** + * Serialize a NarListing to JSON. + */ +void to_json(nlohmann::json & j, const NarListing & listing); + +/** + * Serialize a ShallowNarListing to JSON. + */ +void to_json(nlohmann::json & j, const ShallowNarListing & listing); } // namespace nix diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index 6cc51a7fa..6a9a0772b 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -68,25 +68,26 @@ bool MemorySourceAccessor::pathExists(const CanonPath & path) return open(path, std::nullopt); } -MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const +template<> +SourceAccessor::Stat MemorySourceAccessor::File::lstat() const { return std::visit( overloaded{ [](const Regular & r) { - return Stat{ - .type = tRegular, + return SourceAccessor::Stat{ + .type = SourceAccessor::tRegular, .fileSize = r.contents.size(), .isExecutable = r.executable, }; }, [](const Directory &) { - return Stat{ - .type = tDirectory, + return SourceAccessor::Stat{ + .type = SourceAccessor::tDirectory, }; }, [](const Symlink &) { - return Stat{ - .type = tSymlink, + return SourceAccessor::Stat{ + .type = SourceAccessor::tSymlink, }; }, }, diff --git a/src/libutil/nar-accessor.cc b/src/libutil/nar-accessor.cc index 12db8ac7b..35ee4e536 100644 --- a/src/libutil/nar-accessor.cc +++ b/src/libutil/nar-accessor.cc @@ -272,41 +272,39 @@ GetNarBytes seekableGetNarBytes(const Path & path) }; } -using nlohmann::json; +template +using ListNarResult = std::conditional_t; -json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse) +template +static ListNarResult listNarImpl(SourceAccessor & accessor, const CanonPath & path) { auto st = accessor.lstat(path); - json obj = json::object(); - switch (st.type) { case SourceAccessor::Type::tRegular: - obj["type"] = "regular"; - if (st.fileSize) - obj["size"] = *st.fileSize; - if (st.isExecutable) - obj["executable"] = true; - if (st.narOffset && *st.narOffset) - obj["narOffset"] = *st.narOffset; - break; - case SourceAccessor::Type::tDirectory: - obj["type"] = "directory"; - { - obj["entries"] = json::object(); - json & res2 = obj["entries"]; - for (const auto & [name, type] : accessor.readDirectory(path)) { - if (recurse) { - res2[name] = listNar(accessor, path / name, true); - } else - res2[name] = json::object(); + return typename ListNarResult::Regular{ + .executable = st.isExecutable, + .contents = + NarListingRegularFile{ + .fileSize = st.fileSize, + .narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt, + }, + }; + case SourceAccessor::Type::tDirectory: { + typename ListNarResult::Directory dir; + for (const auto & [name, type] : accessor.readDirectory(path)) { + if constexpr (deep) { + dir.entries.emplace(name, listNarImpl(accessor, path / name)); + } else { + dir.entries.emplace(name, fso::Opaque{}); } } - break; + return dir; + } case SourceAccessor::Type::tSymlink: - obj["type"] = "symlink"; - obj["target"] = accessor.readLink(path); - break; + return typename ListNarResult::Symlink{ + .target = accessor.readLink(path), + }; case SourceAccessor::Type::tBlock: case SourceAccessor::Type::tChar: case SourceAccessor::Type::tSocket: @@ -314,7 +312,64 @@ json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse) case SourceAccessor::Type::tUnknown: assert(false); // cannot happen for NARs } - return obj; +} + +NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path) +{ + return listNarImpl(accessor, path); +} + +ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path) +{ + return listNarImpl(accessor, path); +} + +template +static void to_json_impl(nlohmann::json & j, const Listing & listing) +{ + std::visit( + overloaded{ + [&](const typename Listing::Regular & r) { + j = nlohmann::json::object(); + j["type"] = "regular"; + if (r.contents.fileSize) + j["size"] = *r.contents.fileSize; + if (r.executable) + j["executable"] = true; + if (r.contents.narOffset) + j["narOffset"] = *r.contents.narOffset; + }, + [&](const typename Listing::Directory & d) { + j = nlohmann::json::object(); + j["type"] = "directory"; + j["entries"] = nlohmann::json::object(); + for (const auto & [name, child] : d.entries) { + if constexpr (std::is_same_v) { + to_json(j["entries"][name], child); + } else if constexpr (std::is_same_v) { + j["entries"][name] = nlohmann::json::object(); + } else { + static_assert(false); + } + } + }, + [&](const typename Listing::Symlink & s) { + j = nlohmann::json::object(); + j["type"] = "symlink"; + j["target"] = s.target; + }, + }, + listing.raw); +} + +void to_json(nlohmann::json & j, const NarListing & listing) +{ + to_json_impl(j, listing); +} + +void to_json(nlohmann::json & j, const ShallowNarListing & listing) +{ + to_json_impl(j, listing); } } // namespace nix diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 5c3d7cfd4..812dfdbcf 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -80,7 +80,7 @@ struct CmdCatNar : StoreCommand, MixCat throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); - auto listing = listNar(*narAccessor, CanonPath::root, true); + nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root); cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index ccd479c0a..fd4a98d7a 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -85,7 +85,12 @@ struct MixLs : virtual Args, MixJSON if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(*accessor, path, recursive)); + nlohmann::json j; + if (recursive) + j = listNarDeep(*accessor, path); + else + j = listNarShallow(*accessor, path); + logger->cout("%s", j.dump()); } else listText(accessor, std::move(path)); } @@ -150,7 +155,7 @@ struct CmdLsNar : Command, MixLs throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); - auto listing = listNar(*narAccessor, CanonPath::root, true); + nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root); list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } };