From 64c55961eb52d298e0643481c31fd178a59f5cd4 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Thu, 16 Oct 2025 16:16:54 -0700 Subject: [PATCH] Merge pull request #14273 from fzakaria/fzakaria/issue-13944 Make `nix nar [cat|ls]` lazy --- .../include/nix/store/nar-accessor.hh | 7 +++- src/libstore/nar-accessor.cc | 34 +++++++++++++++---- src/libstore/remote-fs-accessor.cc | 22 ++---------- src/nix/cat.cc | 10 +++++- src/nix/ls.cc | 6 +++- 5 files changed, 50 insertions(+), 29 deletions(-) diff --git a/src/libstore/include/nix/store/nar-accessor.hh b/src/libstore/include/nix/store/nar-accessor.hh index 0e69d436e..bfba5da73 100644 --- a/src/libstore/include/nix/store/nar-accessor.hh +++ b/src/libstore/include/nix/store/nar-accessor.hh @@ -27,7 +27,12 @@ ref makeNarAccessor(Source & source); */ using GetNarBytes = std::function; -ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes); +/** + * The canonical GetNarBytes function for a seekable Source. + */ +GetNarBytes seekableGetNarBytes(const Path & path); + +ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes); /** * Write a JSON representation of the contents of a NAR (except file diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 63fe774c9..f0882d52d 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -141,14 +141,14 @@ struct NarAccessor : public SourceAccessor parseDump(indexer, indexer); } - NarAccessor(const std::string & listing, GetNarBytes getNarBytes) + NarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes) : getNarBytes(getNarBytes) { using json = nlohmann::json; - std::function recurse; + std::function recurse; - recurse = [&](NarMember & member, json & v) { + recurse = [&](NarMember & member, const json & v) { std::string type = v["type"]; if (type == "directory") { @@ -169,8 +169,7 @@ struct NarAccessor : public SourceAccessor return; }; - json v = json::parse(listing); - recurse(root, v); + recurse(root, listing); } NarMember * find(const CanonPath & path) @@ -251,11 +250,34 @@ ref makeNarAccessor(Source & source) return make_ref(source); } -ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) +ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes) { return make_ref(listing, getNarBytes); } +GetNarBytes seekableGetNarBytes(const Path & path) +{ + return [path](uint64_t offset, uint64_t length) { + AutoCloseFD fd = toDescriptor(open( + path.c_str(), + O_RDONLY +#ifndef _WIN32 + | O_CLOEXEC +#endif + )); + if (!fd) + throw SysError("opening NAR cache file '%s'", path); + + if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset) + throw SysError("seeking in '%s'", path); + + std::string buf(length, 0); + readFull(fd.get(), buf.data(), length); + + return buf; + }; +} + using nlohmann::json; json listNar(ref accessor, const CanonPath & path, bool recurse) diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index e6715cbdf..f7ca28ae2 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -70,26 +70,8 @@ std::shared_ptr RemoteFSAccessor::accessObject(const StorePath & try { listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); - - auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) { - AutoCloseFD fd = toDescriptor(open( - cacheFile.c_str(), - O_RDONLY -#ifndef _WIN32 - | O_CLOEXEC -#endif - )); - if (!fd) - throw SysError("opening NAR cache file '%s'", cacheFile); - - if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset) - throw SysError("seeking in '%s'", cacheFile); - - std::string buf(length, 0); - readFull(fd.get(), buf.data(), length); - - return buf; - }); + auto listingJson = nlohmann::json::parse(listing); + auto narAccessor = makeLazyNarAccessor(listingJson, seekableGetNarBytes(cacheFile)); nars.emplace(storePath.hashPart(), narAccessor); return narAccessor; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index effe544e6..5b93d560b 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,6 +1,10 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" #include "nix/store/nar-accessor.hh" +#include "nix/util/serialise.hh" +#include "nix/util/source-accessor.hh" + +#include using namespace nix; @@ -71,7 +75,11 @@ struct CmdCatNar : StoreCommand, MixCat void run(ref store) override { - cat(makeNarAccessor(readFile(narPath)), CanonPath{path}); + AutoCloseFD fd = open(narPath.c_str(), O_RDONLY); + auto source = FdSource{fd.get()}; + auto narAccessor = makeNarAccessor(source); + auto listing = listNar(narAccessor, CanonPath::root, true); + cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 5cdfc2c0f..846af246d 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -145,7 +145,11 @@ struct CmdLsNar : Command, MixLs void run() override { - list(makeNarAccessor(readFile(narPath)), CanonPath{path}); + AutoCloseFD fd = open(narPath.c_str(), O_RDONLY); + auto source = FdSource{fd.get()}; + auto narAccessor = makeNarAccessor(source); + auto listing = listNar(narAccessor, CanonPath::root, true); + list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } };