From 4df60e639b7e492ac5f651f2b3aa02055de5549a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Sep 2025 12:09:46 -0400 Subject: [PATCH 1/7] Use shared pointers in the memory source accessor This allows aliasing, like hard links. --- src/libutil-tests/git.cc | 16 ++++++++-------- .../include/nix/util/memory-source-accessor.hh | 16 ++++++++++++---- src/libutil/memory-source-accessor.cc | 6 +++--- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index 6180a4cfc..a06c5896d 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -233,30 +233,30 @@ TEST_F(GitTest, both_roundrip) .contents{ { "foo", - File::Regular{ + make_ref(File::Regular{ .contents = "hello\n\0\n\tworld!", - }, + }), }, { "bar", - File::Directory{ + make_ref(File::Directory{ .contents = { { "baz", - File::Regular{ + make_ref(File::Regular{ .executable = true, .contents = "good day,\n\0\n\tworld!", - }, + }), }, { "quux", - File::Symlink{ + make_ref(File::Symlink{ .target = "/over/there", - }, + }), }, }, - }, + }), }, }, }; diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index 98c193800..be1d17665 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -35,7 +35,7 @@ struct MemorySourceAccessor : virtual SourceAccessor { using Name = std::string; - std::map> contents; + std::map, std::less<>> contents; bool operator==(const Directory &) const noexcept; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. @@ -89,13 +89,21 @@ struct MemorySourceAccessor : virtual SourceAccessor SourcePath addFile(CanonPath path, std::string && contents); }; -inline bool MemorySourceAccessor::File::Directory::operator==( - const MemorySourceAccessor::File::Directory &) const noexcept = default; +inline bool +MemorySourceAccessor::File::Directory::operator==(const MemorySourceAccessor::File::Directory & other) const noexcept +{ + return std::ranges::equal(contents, other.contents, [](const auto & lhs, const auto & rhs) -> bool { + return lhs.first == rhs.first && *lhs.second == *rhs.second; + }); +}; inline bool MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept { - return contents < other.contents; + return std::ranges::lexicographical_compare( + contents, other.contents, [](const auto & lhs, const auto & rhs) -> bool { + return lhs.first < rhs.first && *lhs.second < *rhs.second; + }); } inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default; diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index 363f52a54..c25079497 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -24,11 +24,11 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, i, { std::string{name}, - File::Directory{}, + make_ref(File::Directory{}), }); } } - cur = &i->second; + cur = &*i->second; } if (newF && create) @@ -92,7 +92,7 @@ MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const Canon if (auto * d = std::get_if(&f->raw)) { DirEntries res; for (auto & [name, file] : d->contents) - res.insert_or_assign(name, file.lstat().type); + res.insert_or_assign(name, file->lstat().type); return res; } else throw Error("file '%s' is not a directory", path); From 02c9ac445ff527a7b4c5105d20d9ab401117dcee Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 20 Sep 2025 11:25:23 +0300 Subject: [PATCH 2/7] libutil: Improve handling of non-directory root in MemorySourceAccessor --- .../nix/util/memory-source-accessor.hh | 2 +- src/libutil/memory-source-accessor.cc | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index be1d17665..53f1b0241 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -58,7 +58,7 @@ struct MemorySourceAccessor : virtual SourceAccessor Stat lstat() const; }; - File root{File::Directory{}}; + std::optional root; bool operator==(const MemorySourceAccessor &) const noexcept = default; diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index c25079497..7d53d6785 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -4,7 +4,22 @@ namespace nix { MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, std::optional create) { - File * cur = &root; + bool hasRoot = root.has_value(); + + // Special handling of root directory. + if (path.isRoot() && !hasRoot) { + if (create) { + root = std::move(*create); + return &root.value(); + } + return nullptr; + } + + // Root does not exist. + if (!hasRoot) + return nullptr; + + File * cur = &root.value(); bool newF = false; @@ -112,6 +127,10 @@ std::string MemorySourceAccessor::readLink(const CanonPath & path) SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) { + // Create root directory automatically if necessary as a convenience. + if (!root && !path.isRoot()) + open(CanonPath::root, File::Directory{}); + auto * f = open(path, File{File::Regular{}}); if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); From 341878ce0fe7d264acc4425d1685f924b17e0b29 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 20 Sep 2025 12:09:45 +0300 Subject: [PATCH 3/7] libstore: Make dummy store also store path info --- src/libstore/dummy-store.cc | 105 ++++++++++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 17 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 1cd1fd08c..12c55472c 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -19,20 +19,43 @@ struct DummyStore : virtual Store ref config; - ref contents; + struct PathInfoAndContents + { + UnkeyedValidPathInfo info; + ref contents; + }; + + /** + * This is map conceptually owns the file system objects for each + * store object. + */ + std::map contents; + + /** + * This view conceptually just borrows the file systems objects of + * each store object from `contents`, and combines them together + * into one store-wide source accessor. + * + * This is needed just in order to implement `Store::getFSAccessor`. + */ + ref wholeStoreView = make_ref(); DummyStore(ref config) : Store{*config} , config(config) - , contents(make_ref()) { - contents->setPathDisplay(config->storeDir); + wholeStoreView->setPathDisplay(config->storeDir); + MemorySink sink{*wholeStoreView}; + sink.createDirectory(CanonPath::root); } void queryPathInfoUncached( const StorePath & path, Callback> callback) noexcept override { - callback(nullptr); + if (auto it = contents.find(path); it != contents.end()) + callback(std::make_shared(StorePath{path}, it->second.info)); + else + callback(nullptr); } /** @@ -50,7 +73,33 @@ struct DummyStore : virtual Store void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override { - unsupported("addToStore"); + if (config->readOnly) + unsupported("addToStore"); + + if (repair) + throw Error("repairing is not supported for '%s' store", config->getHumanReadableURI()); + + if (checkSigs) + throw Error("checking signatures is not supported for '%s' store", config->getHumanReadableURI()); + + auto temp = make_ref(); + MemorySink tempSink{*temp}; + parseDump(tempSink, source); + auto path = info.path; + + auto [it, _] = contents.insert({ + path, + { + std::move(info), + make_ref(std::move(*temp)), + }, + }); + + auto & pathAndContents = it->second; + + bool inserted = wholeStoreView->open(CanonPath(path.to_string()), pathAndContents.contents->root); + if (!inserted) + unreachable(); } StorePath addToStoreFromDump( @@ -65,6 +114,9 @@ struct DummyStore : virtual Store if (config->readOnly) unsupported("addToStoreFromDump"); + if (repair) + throw Error("repairing is not supported for '%s' store", config->getHumanReadableURI()); + auto temp = make_ref(); { @@ -85,22 +137,41 @@ struct DummyStore : virtual Store } auto hash = hashPath({temp, CanonPath::root}, hashMethod.getFileIngestionMethod(), hashAlgo).first; + auto narHash = hashPath({temp, CanonPath::root}, FileIngestionMethod::NixArchive, HashAlgorithm::SHA256); - auto desc = ContentAddressWithReferences::fromParts( - hashMethod, - hash, + auto info = ValidPathInfo::makeFromCA( + *this, + name, + ContentAddressWithReferences::fromParts( + hashMethod, + std::move(hash), + { + .others = references, + // caller is not capable of creating a self-reference, because + // this is content-addressed without modulus + .self = false, + }), + std::move(narHash.first)); + + info.narSize = narHash.second.value(); + + auto path = info.path; + + auto [it, _] = contents.insert({ + path, { - .others = references, - // caller is not capable of creating a self-reference, because - // this is content-addressed without modulus - .self = false, - }); + std::move(info), + make_ref(std::move(*temp)), + }, + }); - auto dstPath = makeFixedOutputPathFromCA(name, desc); + auto & pathAndContents = it->second; - contents->open(CanonPath(printStorePath(dstPath)), std::move(temp->root)); + bool inserted = wholeStoreView->open(CanonPath(path.to_string()), pathAndContents.contents->root); + if (!inserted) + unreachable(); - return dstPath; + return path; } void narFromPath(const StorePath & path, Sink & sink) override @@ -116,7 +187,7 @@ struct DummyStore : virtual Store virtual ref getFSAccessor(bool requireValidPath) override { - return this->contents; + return wholeStoreView; } }; From ed9b377928bf94ae80ef6245f4b50583eacd2db6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 20 Sep 2025 12:10:48 +0300 Subject: [PATCH 4/7] libstore: Disable path info cache for dummy store --- src/libstore/include/nix/store/dummy-store.hh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index 0a15667b6..4898e8a5b 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -4,10 +4,15 @@ namespace nix { struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { - using StoreConfig::StoreConfig; + DummyStoreConfig(const Params & params) + : StoreConfig(params) + { + // Disable caching since this a temporary in-memory store. + pathInfoCacheSize = 0; + } DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) + : DummyStoreConfig(params) { if (!authority.empty()) throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); From 3a4c618483342b64c01e8598ea2d09a4b61e98c3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 20 Sep 2025 12:31:26 +0300 Subject: [PATCH 5/7] libstore: Fix typo in description of dummy store --- src/libstore/dummy-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/dummy-store.md b/src/libstore/dummy-store.md index 3cbec3b3a..3ba96fecb 100644 --- a/src/libstore/dummy-store.md +++ b/src/libstore/dummy-store.md @@ -4,7 +4,7 @@ R"( This store type represents a store in memory. Store objects can be read and written, but only so long as the store is open. -Once the store is closed, all data will be forgoton. +Once the store is closed, all data will be discarded. It's useful when you want to use the Nix evaluator when no actual Nix store exists, e.g. From b66c357b5833f8e44ca12de7a766ef8691d6279e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 20 Sep 2025 17:36:35 +0300 Subject: [PATCH 6/7] libstore: Implement DummyStore::narFromPath --- src/libstore/dummy-store.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 12c55472c..06b518c15 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -176,7 +176,13 @@ struct DummyStore : virtual Store void narFromPath(const StorePath & path, Sink & sink) override { - unsupported("narFromPath"); + auto object = contents.find(path); + if (object == contents.end()) + throw Error("path '%s' is not valid", printStorePath(path)); + + const auto & [info, accessor] = object->second; + SourcePath sourcePath(accessor); + dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive); } void From a453a49043999fcdb726e5d95634914de9234fb7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 21 Sep 2025 13:36:31 +0300 Subject: [PATCH 7/7] tests: Tests for writeable dummy in-memory store --- tests/functional/eval-store.sh | 4 ++++ tests/functional/flakes/flakes.sh | 1 + tests/nixos/github-flakes.nix | 1 + 3 files changed, 6 insertions(+) diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh index 202e7b004..92faa4005 100755 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -52,3 +52,7 @@ rm -rf "$eval_store" [[ $(nix eval --eval-store "$eval_store?require-sigs=false" --impure --raw --file ./ifd.nix) = hi ]] ls $NIX_STORE_DIR/*dependencies-top/foobar (! ls $eval_store/nix/store/*dependencies-top/foobar) + +# Can't write .drv by default +(! nix-instantiate dependencies.nix --eval-store "dummy://") +nix-instantiate dependencies.nix --eval-store "dummy://?read-only=false" diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 7b5be112e..97d238654 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -406,6 +406,7 @@ nix flake update flake1 flake2/flake1 --flake "$flake3Dir" # Test 'nix flake metadata --json'. nix flake metadata "$flake3Dir" --json | jq . +nix flake metadata "$flake3Dir" --json --eval-store "dummy://?read-only=false" | jq . # Test flake in store does not evaluate. rm -rf $badFlakeDir diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 91fd6b062..d14cd9d0c 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -219,6 +219,7 @@ in client.succeed("nix registry pin nixpkgs") client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") + client.succeed("nix eval nixpkgs#hello --eval-store dummy://?read-only=false >&2") # Test fetchTree on a github URL. hash = client.succeed(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree {info['url']}).narHash'")