diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc index 9cecd573e..f2c6fb51e 100644 --- a/src/libstore-tests/references.cc +++ b/src/libstore-tests/references.cc @@ -99,7 +99,7 @@ TEST(references, scanForReferencesDeep) // Create an in-memory file system with various reference patterns auto accessor = make_ref(); accessor->root = File::Directory{ - .contents{ + .entries{ { // file1.txt: contains hash1 "file1.txt", @@ -125,7 +125,7 @@ TEST(references, scanForReferencesDeep) // subdir: a subdirectory "subdir", File::Directory{ - .contents{ + .entries{ { // subdir/file4.txt: contains hash1 again "file4.txt", diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 274e47271..caae72479 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -8,7 +8,7 @@ #include "nix/util/sync.hh" #include "nix/store/remote-fs-accessor.hh" #include "nix/store/nar-info-disk-cache.hh" -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include "nix/util/thread-pool.hh" #include "nix/util/callback.hh" #include "nix/util/signals.hh" @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(ref(narAccessor), CanonPath::root, true)}, + {"root", listNarDeep(*narAccessor, CanonPath::root)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index 5d6626ff8..c17d6a9cb 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -55,7 +55,6 @@ headers = [ config_pub_h ] + files( 'machines.hh', 'make-content-addressed.hh', 'names.hh', - 'nar-accessor.hh', 'nar-info-disk-cache.hh', 'nar-info.hh', 'outputs-spec.hh', diff --git a/src/libstore/include/nix/store/nar-accessor.hh b/src/libstore/include/nix/store/nar-accessor.hh deleted file mode 100644 index bfba5da73..000000000 --- a/src/libstore/include/nix/store/nar-accessor.hh +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once -///@file - -#include "nix/util/source-accessor.hh" - -#include - -#include - -namespace nix { - -struct Source; - -/** - * Return an object that provides access to the contents of a NAR - * file. - */ -ref makeNarAccessor(std::string && nar); - -ref makeNarAccessor(Source & source); - -/** - * Create a NAR accessor from a NAR listing (in the format produced by - * listNar()). The callback getNarBytes(offset, length) is used by the - * readFile() method of the accessor to get the contents of files - * inside the NAR. - */ -using GetNarBytes = std::function; - -/** - * The canonical GetNarBytes function for a seekable Source. - */ -GetNarBytes seekableGetNarBytes(const Path & path); - -ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes); - -/** - * Write a JSON representation of the contents of a NAR (except file - * contents). - */ -nlohmann::json listNar(ref accessor, const CanonPath & path, bool recurse); - -} // namespace nix diff --git a/src/libstore/meson.build b/src/libstore/meson.build index d1b3666cc..e3425deb5 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -300,7 +300,6 @@ sources = files( 'make-content-addressed.cc', 'misc.cc', 'names.cc', - 'nar-accessor.cc', 'nar-info-disk-cache.cc', 'nar-info.cc', 'optimise-store.cc', diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index f7ca28ae2..51bab9953 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -1,6 +1,6 @@ #include #include "nix/store/remote-fs-accessor.hh" -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include #include @@ -39,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std: if (cacheDir != "") { try { - nlohmann::json j = listNar(narAccessor, CanonPath::root, true); + nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreExceptionExceptInterrupt(); diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index 6180a4cfc..9d749b492 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -230,7 +230,7 @@ TEST_F(GitTest, both_roundrip) auto files = make_ref(); files->root = File::Directory{ - .contents{ + .entries{ { "foo", File::Regular{ @@ -240,7 +240,7 @@ TEST_F(GitTest, both_roundrip) { "bar", File::Directory{ - .contents = + .entries = { { "baz", diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index eba282fe1..268f6b06f 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -4,59 +4,111 @@ #include "nix/util/source-path.hh" #include "nix/util/fs-sink.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { +/** + * File System Object definitions + * + * @see https://nix.dev/manual/nix/latest/store/file-system-object.html + */ +namespace fso { + +template +struct Regular +{ + bool executable = false; + RegularContents contents; + + auto operator<=>(const Regular &) const = default; +}; + +/** + * Child parameter because sometimes we want "shallow" directories without + * full file children. + */ +template +struct DirectoryT +{ + using Name = std::string; + + std::map> entries; + + inline bool operator==(const DirectoryT &) const noexcept; + inline std::strong_ordering operator<=>(const DirectoryT &) const noexcept; +}; + +struct Symlink +{ + std::string target; + + auto operator<=>(const Symlink &) const = default; +}; + +/** + * For when we know there is child, but don't know anything about it. + * + * This is not part of the core File System Object data model --- this + * represents not knowing, not an additional type of file. + */ +struct Opaque +{ + auto operator<=>(const Opaque &) const = default; +}; + +/** + * `File` nicely defining what a "file system object" + * is in Nix. + * + * With a different type arugment, it is also can be a "skeletal" + * version is that abstract syntax for a "NAR listing". + */ +template +struct VariantT +{ + bool operator==(const VariantT &) const noexcept; + std::strong_ordering operator<=>(const VariantT &) const noexcept; + + using Regular = nix::fso::Regular; + + /** + * In the default case, we do want full file children for our directory. + */ + using Directory = nix::fso::DirectoryT>; + + using Symlink = nix::fso::Symlink; + + using Raw = std::variant; + Raw raw; + + MAKE_WRAPPER_CONSTRUCTOR(VariantT); + + SourceAccessor::Stat lstat() const; +}; + +template +inline bool DirectoryT::operator==(const DirectoryT &) const noexcept = default; + +template +inline std::strong_ordering DirectoryT::operator<=>(const DirectoryT &) const noexcept = default; + +template +inline bool +VariantT::operator==(const VariantT &) const noexcept = default; + +template +inline std::strong_ordering +VariantT::operator<=>(const VariantT &) const noexcept = default; + +} // namespace fso + /** * An source accessor for an in-memory file system. */ struct MemorySourceAccessor : virtual SourceAccessor { - /** - * In addition to being part of the implementation of - * `MemorySourceAccessor`, this has a side benefit of nicely - * defining what a "file system object" is in Nix. - */ - struct File - { - bool operator==(const File &) const noexcept; - std::strong_ordering operator<=>(const File &) const noexcept; - - struct Regular - { - bool executable = false; - std::string contents; - - bool operator==(const Regular &) const = default; - auto operator<=>(const Regular &) const = default; - }; - - struct Directory - { - using Name = std::string; - - std::map> contents; - - bool operator==(const Directory &) const noexcept; - // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. - bool operator<(const Directory &) const noexcept; - }; - - struct Symlink - { - std::string target; - - bool operator==(const Symlink &) const = default; - auto operator<=>(const Symlink &) const = default; - }; - - using Raw = std::variant; - Raw raw; - - MAKE_WRAPPER_CONSTRUCTOR(File); - - Stat lstat() const; - }; + using File = fso::VariantT; std::optional root; @@ -89,19 +141,6 @@ struct MemorySourceAccessor : virtual SourceAccessor SourcePath addFile(CanonPath path, std::string && contents); }; -inline bool MemorySourceAccessor::File::Directory::operator==( - const MemorySourceAccessor::File::Directory &) const noexcept = default; - -inline bool -MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept -{ - return contents < other.contents; -} - -inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default; -inline std::strong_ordering -MemorySourceAccessor::File::operator<=>(const MemorySourceAccessor::File &) const noexcept = default; - /** * Write to a `MemorySourceAccessor` at the given path */ diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 9a606e15d..b6677140e 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -50,6 +50,7 @@ headers = files( 'memory-source-accessor.hh', 'mounted-source-accessor.hh', 'muxable-pipe.hh', + 'nar-accessor.hh', 'os-string.hh', 'pool.hh', 'pos-idx.hh', diff --git a/src/libutil/include/nix/util/nar-accessor.hh b/src/libutil/include/nix/util/nar-accessor.hh new file mode 100644 index 000000000..9665af9bc --- /dev/null +++ b/src/libutil/include/nix/util/nar-accessor.hh @@ -0,0 +1,88 @@ +#pragma once +///@file + +#include "nix/util/memory-source-accessor.hh" + +#include + +#include + +namespace nix { + +struct Source; + +/** + * Return an object that provides access to the contents of a NAR + * file. + */ +ref makeNarAccessor(std::string && nar); + +ref makeNarAccessor(Source & source); + +/** + * Create a NAR accessor from a NAR listing (in the format produced by + * listNar()). The callback getNarBytes(offset, length) is used by the + * readFile() method of the accessor to get the contents of files + * inside the NAR. + */ +using GetNarBytes = std::function; + +/** + * The canonical GetNarBytes function for a seekable Source. + */ +GetNarBytes seekableGetNarBytes(const Path & path); + +ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes); + +struct NarListingRegularFile +{ + /** + * @see `SourceAccessor::Stat::fileSize` + */ + std::optional fileSize; + + /** + * @see `SourceAccessor::Stat::narOffset` + * + * We only set to non-`std::nullopt` if it is also non-zero. + */ + std::optional narOffset; + + auto operator<=>(const NarListingRegularFile &) const = default; +}; + +/** + * Abstract syntax for a "NAR listing". + */ +using NarListing = fso::VariantT; + +/** + * Shallow NAR listing where directory children are not recursively expanded. + * Uses a variant that can hold Regular/Symlink fully, but Directory children + * are just unit types indicating presence without content. + */ +using ShallowNarListing = fso::VariantT; + +/** + * Return a deep structured representation of the contents of a NAR (except file + * contents), recursively listing all children. + */ +NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path); + +/** + * Return a shallow structured representation of the contents of a NAR (except file + * contents), only listing immediate children without recursing. + */ +ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path); + +/** + * Serialize a NarListing to JSON. + */ +void to_json(nlohmann::json & j, const NarListing & listing); + +/** + * Serialize a ShallowNarListing to JSON. + */ +void to_json(nlohmann::json & j, const ShallowNarListing & listing); + +} // namespace nix diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index a9ffb7746..6a9a0772b 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -29,13 +29,13 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, return nullptr; auto & curDir = *curDirP; - auto i = curDir.contents.find(name); - if (i == curDir.contents.end()) { + auto i = curDir.entries.find(name); + if (i == curDir.entries.end()) { if (!create) return nullptr; else { newF = true; - i = curDir.contents.insert( + i = curDir.entries.insert( i, { std::string{name}, @@ -68,25 +68,26 @@ bool MemorySourceAccessor::pathExists(const CanonPath & path) return open(path, std::nullopt); } -MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const +template<> +SourceAccessor::Stat MemorySourceAccessor::File::lstat() const { return std::visit( overloaded{ [](const Regular & r) { - return Stat{ - .type = tRegular, + return SourceAccessor::Stat{ + .type = SourceAccessor::tRegular, .fileSize = r.contents.size(), .isExecutable = r.executable, }; }, [](const Directory &) { - return Stat{ - .type = tDirectory, + return SourceAccessor::Stat{ + .type = SourceAccessor::tDirectory, }; }, [](const Symlink &) { - return Stat{ - .type = tSymlink, + return SourceAccessor::Stat{ + .type = SourceAccessor::tSymlink, }; }, }, @@ -106,7 +107,7 @@ MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const Canon throw Error("file '%s' does not exist", path); if (auto * d = std::get_if(&f->raw)) { DirEntries res; - for (auto & [name, file] : d->contents) + for (auto & [name, file] : d->entries) res.insert_or_assign(name, file.lstat().type); return res; } else diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8b7a5d977..5290ff2d0 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -147,6 +147,7 @@ sources = [ config_priv_h ] + files( 'logging.cc', 'memory-source-accessor.cc', 'mounted-source-accessor.cc', + 'nar-accessor.cc', 'pos-table.cc', 'position.cc', 'posix-source-accessor.cc', diff --git a/src/libstore/nar-accessor.cc b/src/libutil/nar-accessor.cc similarity index 71% rename from src/libstore/nar-accessor.cc rename to src/libutil/nar-accessor.cc index 640b77540..35ee4e536 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libutil/nar-accessor.cc @@ -1,4 +1,4 @@ -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include "nix/util/archive.hh" #include @@ -272,41 +272,39 @@ GetNarBytes seekableGetNarBytes(const Path & path) }; } -using nlohmann::json; +template +using ListNarResult = std::conditional_t; -json listNar(ref accessor, const CanonPath & path, bool recurse) +template +static ListNarResult listNarImpl(SourceAccessor & accessor, const CanonPath & path) { - auto st = accessor->lstat(path); - - json obj = json::object(); + auto st = accessor.lstat(path); switch (st.type) { case SourceAccessor::Type::tRegular: - obj["type"] = "regular"; - if (st.fileSize) - obj["size"] = *st.fileSize; - if (st.isExecutable) - obj["executable"] = true; - if (st.narOffset && *st.narOffset) - obj["narOffset"] = *st.narOffset; - break; - case SourceAccessor::Type::tDirectory: - obj["type"] = "directory"; - { - obj["entries"] = json::object(); - json & res2 = obj["entries"]; - for (const auto & [name, type] : accessor->readDirectory(path)) { - if (recurse) { - res2[name] = listNar(accessor, path / name, true); - } else - res2[name] = json::object(); + return typename ListNarResult::Regular{ + .executable = st.isExecutable, + .contents = + NarListingRegularFile{ + .fileSize = st.fileSize, + .narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt, + }, + }; + case SourceAccessor::Type::tDirectory: { + typename ListNarResult::Directory dir; + for (const auto & [name, type] : accessor.readDirectory(path)) { + if constexpr (deep) { + dir.entries.emplace(name, listNarImpl(accessor, path / name)); + } else { + dir.entries.emplace(name, fso::Opaque{}); } } - break; + return dir; + } case SourceAccessor::Type::tSymlink: - obj["type"] = "symlink"; - obj["target"] = accessor->readLink(path); - break; + return typename ListNarResult::Symlink{ + .target = accessor.readLink(path), + }; case SourceAccessor::Type::tBlock: case SourceAccessor::Type::tChar: case SourceAccessor::Type::tSocket: @@ -314,7 +312,64 @@ json listNar(ref accessor, const CanonPath & path, bool recurse) case SourceAccessor::Type::tUnknown: assert(false); // cannot happen for NARs } - return obj; +} + +NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path) +{ + return listNarImpl(accessor, path); +} + +ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path) +{ + return listNarImpl(accessor, path); +} + +template +static void to_json_impl(nlohmann::json & j, const Listing & listing) +{ + std::visit( + overloaded{ + [&](const typename Listing::Regular & r) { + j = nlohmann::json::object(); + j["type"] = "regular"; + if (r.contents.fileSize) + j["size"] = *r.contents.fileSize; + if (r.executable) + j["executable"] = true; + if (r.contents.narOffset) + j["narOffset"] = *r.contents.narOffset; + }, + [&](const typename Listing::Directory & d) { + j = nlohmann::json::object(); + j["type"] = "directory"; + j["entries"] = nlohmann::json::object(); + for (const auto & [name, child] : d.entries) { + if constexpr (std::is_same_v) { + to_json(j["entries"][name], child); + } else if constexpr (std::is_same_v) { + j["entries"][name] = nlohmann::json::object(); + } else { + static_assert(false); + } + } + }, + [&](const typename Listing::Symlink & s) { + j = nlohmann::json::object(); + j["type"] = "symlink"; + j["target"] = s.target; + }, + }, + listing.raw); +} + +void to_json(nlohmann::json & j, const NarListing & listing) +{ + to_json_impl(j, listing); +} + +void to_json(nlohmann::json & j, const ShallowNarListing & listing) +{ + to_json_impl(j, listing); } } // namespace nix diff --git a/src/nix/cat.cc b/src/nix/cat.cc index bf58bb492..812dfdbcf 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include "nix/util/serialise.hh" #include "nix/util/source-accessor.hh" @@ -80,7 +80,7 @@ struct CmdCatNar : StoreCommand, MixCat throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); - auto listing = listNar(narAccessor, CanonPath::root, true); + nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root); cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 9bf3c5996..fd4a98d7a 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include "nix/main/common-args.hh" #include @@ -85,7 +85,12 @@ struct MixLs : virtual Args, MixJSON if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(accessor, path, recursive)); + nlohmann::json j; + if (recursive) + j = listNarDeep(*accessor, path); + else + j = listNarShallow(*accessor, path); + logger->cout("%s", j.dump()); } else listText(accessor, std::move(path)); } @@ -150,7 +155,7 @@ struct CmdLsNar : Command, MixLs throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); - auto listing = listNar(narAccessor, CanonPath::root, true); + nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root); list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } };