1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-08 19:46:02 +01:00

Merge pull request #14050 from NixOS/fix-fetch-to-store-caching

Fix fetchToStore caching
This commit is contained in:
Eelco Dolstra 2025-10-06 19:39:41 +02:00 committed by GitHub
commit 1e709554d5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 188 additions and 92 deletions

View file

@ -236,24 +236,17 @@ EvalState::EvalState(
{CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
}))
, rootFS([&] {
auto accessor = [&]() -> decltype(rootFS) {
/* In pure eval mode, we provide a filesystem that only
contains the Nix store. */
if (settings.pureEval)
return storeFS;
contains the Nix store.
/* If we have a chroot store and pure eval is not enabled,
use a union accessor to make the chroot store available
at its logical location while still having the underlying
directory available. This is necessary for instance if
we're evaluating a file from the physical /nix/store
while using a chroot store. */
auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
if (store->storeDir != realStoreDir)
return makeUnionSourceAccessor({getFSSourceAccessor(), storeFS});
return getFSSourceAccessor();
}();
Otherwise, use a union accessor to make the augmented store
available at its logical location while still having the
underlying directory available. This is necessary for
instance if we're evaluating a file from the physical
/nix/store while using a chroot store, and also for lazy
mounted fetchTree. */
auto accessor = settings.pureEval ? storeFS.cast<SourceAccessor>()
: makeUnionSourceAccessor({getFSSourceAccessor(), storeFS});
/* Apply access control if needed. */
if (settings.restrictEval || settings.pureEval)
@ -3133,6 +3126,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
auto res = (r / CanonPath(suffix)).resolveSymlinks();
if (res.pathExists())
return res;
// Backward compatibility hack: throw an exception if access
// to this path is not allowed.
if (auto accessor = res.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
accessor->checkAccess(res.path);
}
if (hasPrefix(path, "nix/"))
@ -3199,6 +3197,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
if (path.resolveSymlinks().pathExists())
return finish(std::move(path));
else {
// Backward compatibility hack: throw an exception if access
// to this path is not allowed.
if (auto accessor = path.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
accessor->checkAccess(path.path);
logWarning({.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)});
}
}

View file

@ -42,6 +42,7 @@ class Store;
namespace fetchers {
struct Settings;
struct InputCache;
struct Input;
} // namespace fetchers
struct EvalSettings;
class EvalState;
@ -575,6 +576,11 @@ public:
void checkURI(const std::string & uri);
/**
* Mount an input on the Nix store.
*/
StorePath mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor);
/**
* Parse a Nix expression from the specified file.
*/

View file

@ -1,5 +1,7 @@
#include "nix/store/store-api.hh"
#include "nix/expr/eval.hh"
#include "nix/util/mounted-source-accessor.hh"
#include "nix/fetchers/fetch-to-store.hh"
namespace nix {
@ -18,4 +20,27 @@ SourcePath EvalState::storePath(const StorePath & path)
return {rootFS, CanonPath{store->printStorePath(path)}};
}
StorePath
EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
{
auto storePath = fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
allowPath(storePath); // FIXME: should just whitelist the entire virtual store
storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
auto narHash = store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
if (originalInput.getNarHash() && narHash != *originalInput.getNarHash())
throw Error(
(unsigned int) 102,
"NAR hash mismatch in input '%s', expected '%s' but got '%s'",
originalInput.to_string(),
narHash.to_string(HashFormat::SRI, true),
originalInput.getNarHash()->to_string(HashFormat::SRI, true));
return storePath;
}
} // namespace nix

View file

@ -10,6 +10,7 @@
#include "nix/util/url.hh"
#include "nix/expr/value-to-json.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/input-cache.hh"
#include <nlohmann/json.hpp>
@ -218,11 +219,11 @@ static void fetchTree(
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
}
auto [storePath, input2] = input.fetchToStore(state.store);
auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No);
state.allowPath(storePath);
auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false);
}
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, Value & v)

View file

@ -1,6 +1,7 @@
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/fetchers.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/util/environment-variables.hh"
namespace nix {
@ -27,14 +28,22 @@ StorePath fetchToStore(
std::optional<fetchers::Cache::Key> cacheKey;
if (!filter && path.accessor->fingerprint) {
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
auto [subpath, fingerprint] = filter ? std::pair<CanonPath, std::optional<std::string>>{path.path, std::nullopt}
: path.accessor->getFingerprint(path.path);
if (fingerprint) {
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs());
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
debug("store path cache hit for '%s'", path);
return res->storePath;
}
} else
} else {
static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
if (barf && !filter)
throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter);
// FIXME: could still provide in-memory caching keyed on `SourcePath`.
debug("source path '%s' is uncacheable", path);
}
Activity act(
*logger,

View file

@ -356,8 +356,10 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
auto [accessor, result] = scheme->getAccessor(store, *this);
assert(!accessor->fingerprint);
if (!accessor->fingerprint)
accessor->fingerprint = result.getFingerprint(store);
else
result.cachedFingerprint = accessor->fingerprint;
return {accessor, std::move(result)};
}

View file

@ -16,15 +16,26 @@ std::string FilteringSourceAccessor::readFile(const CanonPath & path)
return next->readFile(prefix / path);
}
void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
{
checkAccess(path);
return next->readFile(prefix / path, sink, sizeCallback);
}
bool FilteringSourceAccessor::pathExists(const CanonPath & path)
{
return isAllowed(path) && next->pathExists(prefix / path);
}
std::optional<SourceAccessor::Stat> FilteringSourceAccessor::maybeLstat(const CanonPath & path)
{
return isAllowed(path) ? next->maybeLstat(prefix / path) : std::nullopt;
}
SourceAccessor::Stat FilteringSourceAccessor::lstat(const CanonPath & path)
{
checkAccess(path);
return next->maybeLstat(prefix / path);
return next->lstat(prefix / path);
}
SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path)
@ -49,6 +60,13 @@ std::string FilteringSourceAccessor::showPath(const CanonPath & path)
return displayPrefix + next->showPath(prefix / path) + displaySuffix;
}
std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFingerprint(const CanonPath & path)
{
if (fingerprint)
return {path, fingerprint};
return next->getFingerprint(prefix / path);
}
void FilteringSourceAccessor::checkAccess(const CanonPath & path)
{
if (!isAllowed(path))

View file

@ -893,8 +893,7 @@ struct GitInputScheme : InputScheme
return makeFingerprint(*rev);
else {
auto repoInfo = getRepoInfo(input);
if (auto repoPath = repoInfo.getPath();
repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) {
if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) {
/* Calculate a fingerprint that takes into account the
deleted and modified/added files. */
HashSink hashSink{HashAlgorithm::SHA512};
@ -907,7 +906,7 @@ struct GitInputScheme : InputScheme
writeString("deleted:", hashSink);
writeString(file.abs(), hashSink);
}
return makeFingerprint(*repoInfo.workdirInfo.headRev)
return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev))
+ ";d=" + hashSink.finish().hash.to_string(HashFormat::Base16, false);
}
return std::nullopt;

View file

@ -36,8 +36,12 @@ struct FilteringSourceAccessor : SourceAccessor
std::string readFile(const CanonPath & path) override;
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
bool pathExists(const CanonPath & path) override;
Stat lstat(const CanonPath & path) override;
std::optional<Stat> maybeLstat(const CanonPath & path) override;
DirEntries readDirectory(const CanonPath & path) override;
@ -46,6 +50,8 @@ struct FilteringSourceAccessor : SourceAccessor
std::string showPath(const CanonPath & path) override;
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
/**
* Call `makeNotAllowedError` to throw a `RestrictedPathError`
* exception if `isAllowed()` returns `false` for `path`.

View file

@ -123,8 +123,6 @@ struct PathInputScheme : InputScheme
auto absPath = getAbsPath(input);
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
// FIXME: check whether access to 'path' is allowed.
auto storePath = store->maybeParseStorePath(absPath.string());
@ -133,43 +131,33 @@ struct PathInputScheme : InputScheme
time_t mtime = 0;
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
// FIXME: try to substitute storePath.
auto src = sinkToSource(
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
storePath = store->addToStoreFromDump(*src, "source");
}
// To avoid copying the path again to the /nix/store, we need to add a cache entry.
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto fp = getFingerprint(store, input);
if (fp) {
auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/");
input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath);
}
auto accessor = ref{store->getFSAccessor(*storePath)};
// To prevent `fetchToStore()` copying the path again to Nix
// store, pre-create an entry in the fetcher cache.
auto info = store->queryPathInfo(*storePath);
accessor->fingerprint =
fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
input.settings->getCache()->upsert(
makeFetchToStoreCacheKey(
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
*store,
{},
*storePath);
/* Trust the lastModified value supplied by the user, if
any. It's not a "secure" attribute so we don't care. */
if (!input.getLastModified())
input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
return {ref{store->getFSAccessor(*storePath)}, std::move(input)};
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (isRelative(input))
return std::nullopt;
/* If this path is in the Nix store, use the hash of the
store object and the subpath. */
auto path = getAbsPath(input);
try {
auto [storePath, subPath] = store->toStorePath(path.string());
auto info = store->queryPathInfo(storePath);
return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath);
} catch (Error &) {
return std::nullopt;
}
return {accessor, std::move(input)};
}
std::optional<ExperimentalFeature> experimentalFeature() const override

View file

@ -24,21 +24,6 @@ using namespace flake;
namespace flake {
static StorePath copyInputToStore(
EvalState & state, fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
{
auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName());
state.allowPath(storePath);
auto narHash = state.store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
return storePath;
}
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
{
if (value.isThunk() && value.isTrivial())
@ -360,11 +345,14 @@ static Flake getFlake(
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
}
// Copy the tree to the store.
auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, cachedInput.accessor);
// Re-parse flake.nix from the store.
return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath);
return readFlake(
state,
originalRef,
resolvedRef,
lockedRef,
state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)),
lockRootAttrPath);
}
Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries)
@ -721,11 +709,10 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
// FIXME: allow input to be lazy.
auto storePath = copyInputToStore(
state, lockedRef.input, input.ref->input, cachedInput.accessor);
return {state.storePath(storePath), lockedRef};
return {
state.storePath(
state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor)),
lockedRef};
}
}();

View file

@ -121,7 +121,7 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
std::string typeString();
};
Stat lstat(const CanonPath & path);
virtual Stat lstat(const CanonPath & path);
virtual std::optional<Stat> maybeLstat(const CanonPath & path) = 0;
@ -180,6 +180,27 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
*/
std::optional<std::string> fingerprint;
/**
* Return the fingerprint for `path`. This is usually the
* fingerprint of the current accessor, but for composite
* accessors (like `MountedSourceAccessor`), we want to return the
* fingerprint of the "inner" accessor if the current one lacks a
* fingerprint.
*
* So this method is intended to return the most-outer accessor
* that has a fingerprint for `path`. It also returns the path that `path`
* corresponds to in that accessor.
*
* For example: in a `MountedSourceAccessor` that has
* `/nix/store/foo` mounted,
* `getFingerprint("/nix/store/foo/bar")` will return the path
* `/bar` and the fingerprint of the `/nix/store/foo` accessor.
*/
virtual std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path)
{
return {path, fingerprint};
}
/**
* Return the maximum last-modified time of the files in this
* tree, if available.

View file

@ -27,6 +27,12 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor
return accessor->readFile(subpath);
}
Stat lstat(const CanonPath & path) override
{
auto [accessor, subpath] = resolve(path);
return accessor->lstat(subpath);
}
std::optional<Stat> maybeLstat(const CanonPath & path) override
{
auto [accessor, subpath] = resolve(path);
@ -85,6 +91,14 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor
else
return nullptr;
}
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override
{
if (fingerprint)
return {path, fingerprint};
auto [accessor, subpath] = resolve(path);
return accessor->getFingerprint(subpath);
}
};
ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts)

View file

@ -72,6 +72,18 @@ struct UnionSourceAccessor : SourceAccessor
}
return std::nullopt;
}
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override
{
if (fingerprint)
return {path, fingerprint};
for (auto & accessor : accessors) {
auto [subpath, fingerprint] = accessor->getFingerprint(path);
if (fingerprint)
return {subpath, fingerprint};
}
return {path, std::nullopt};
}
};
ref<SourceAccessor> makeUnionSourceAccessor(std::vector<ref<SourceAccessor>> && accessors)

View file

@ -2,6 +2,8 @@
source ../common.sh
export _NIX_TEST_BARF_ON_UNCACHEABLE=1
# shellcheck disable=SC2034 # this variable is used by tests that source this file
registry=$TEST_ROOT/registry.json

View file

@ -62,8 +62,8 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule
# Check that dirtying a submodule makes the entire thing dirty.
[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]]
echo '"foo"' > "$rootRepo"/submodule/sub.nix
[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]]
[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]]
[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval --json "$flakeref#sub" ) = '"foo"' ]]
[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]]
# Test that `nix flake metadata` parses `submodule` correctly.
cat > "$rootRepo"/flake.nix <<EOF
@ -75,7 +75,7 @@ EOF
git -C "$rootRepo" add flake.nix
git -C "$rootRepo" commit -m "Add flake.nix"
storePath=$(nix flake prefetch --json "$rootRepo?submodules=1" | jq -r .storePath)
storePath=$(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake prefetch --json "$rootRepo?submodules=1" | jq -r .storePath)
[[ -e "$storePath/submodule" ]]
# Test the use of inputs.self.

View file

@ -131,7 +131,7 @@ EOF
git -C "$flakeFollowsA" add flake.nix
expect 1 nix flake lock "$flakeFollowsA" 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode'
expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep '/flakeB.*does not exist'
expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep "'flakeB' is too short to be a valid store path"
# Test relative non-flake inputs.
cat > "$flakeFollowsA"/flake.nix <<EOF

View file

@ -27,9 +27,9 @@ nix build -o "$TEST_ROOT/result" "hg+file://$flake2Dir"
(! nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revision)
nix eval "hg+file://$flake2Dir"#expr
_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr
nix eval "hg+file://$flake2Dir"#expr
_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr
(! nix eval "hg+file://$flake2Dir"#expr --no-allow-dirty)

View file

@ -81,7 +81,7 @@ nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file
nix registry add --registry "$registry" flake3 "git+file://$flake3Dir"
nix build -o "$TEST_ROOT/result" flake3#fnord
_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord
[[ $(cat "$TEST_ROOT/result") = FNORD ]]
# Check whether flake input fetching is lazy: flake3#sth does not
@ -91,16 +91,17 @@ clearStore
mv "$flake2Dir" "$flake2Dir.tmp"
mv "$nonFlakeDir" "$nonFlakeDir.tmp"
nix build -o "$TEST_ROOT/result" flake3#sth
(! nix build -o "$TEST_ROOT/result" flake3#xyzzy)
(! nix build -o "$TEST_ROOT/result" flake3#fnord)
(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy)
(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord)
mv "$flake2Dir.tmp" "$flake2Dir"
mv "$nonFlakeDir.tmp" "$nonFlakeDir"
nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord
_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord
# Check non-flake inputs have a sourceInfo and an outPath
#
# This may look redundant, but the other checks below happen in a command
# substitution subshell, so failures there will not exit this shell
export _NIX_TEST_BARF_ON_UNCACHEABLE='' # FIXME
nix eval --raw flake3#inputs.nonFlake.outPath
nix eval --raw flake3#inputs.nonFlake.sourceInfo.outPath
nix eval --raw flake3#inputs.nonFlakeFile.outPath

View file

@ -4,6 +4,8 @@ source ./common.sh
requireGit
unset _NIX_TEST_BARF_ON_UNCACHEABLE
# Test a "vendored" subflake dependency. This is a relative path flake
# which doesn't reference the root flake and has its own lock file.
#

View file

@ -10,4 +10,4 @@ error:
… while calling the 'hashFile' builtin
error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory
error: path '/pwd/lang/this-file-is-definitely-not-there-7392097' does not exist