nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-08 18:11:02 +01:00
Code Activity

fetchToStore() cache: Use content hashes instead of store paths

Browse source 
We can always compute the store path from the content hash, but not
vice versa. Storing the content hash allows `hashPath()` to be
replaced by `fetchToStore(...FetchMode::DryRun...)`, which gets us
caching in lazy-trees mode.
This commit is contained in:
Eelco Dolstra 2025-06-13 22:19:53 +02:00 • committed by Graham Christensen
parent b0aa12df25
commit dc5e620032
6 changed files with 72 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

73
src/libfetchers/fetch-to-store.cc
View file

@ -3,19 +3,16 @@
namespace nix {
fetchers::Cache::Key makeFetchToStoreCacheKey(
const std::string &name,
const std::string &fingerprint,
fetchers::Cache::Key makeSourcePathToHashCacheKey(
const std::string & fingerprint,
ContentAddressMethod method,
const std::string &path)
const std::string & path)
{
return fetchers::Cache::Key{"fetchToStore", {
{"name", name},
return fetchers::Cache::Key{"sourcePathToHash", {
{"fingerprint", fingerprint},
{"method", std::string{method.render()}},
{"path", path}
}};
}
StorePath fetchToStore(
@ -27,9 +24,18 @@ StorePath fetchToStore(
PathFilter * filter,
RepairFlag repair)
{
// FIXME: add an optimisation for the case where the accessor is
// a `PosixSourceAccessor` pointing to a store path.
return fetchToStore2(store, path, mode, name, method, filter, repair).first;
}
std::pair<StorePath, Hash> fetchToStore2(
Store & store,
const SourcePath & path,
FetchMode mode,
std::string_view name,
ContentAddressMethod method,
PathFilter * filter,
RepairFlag repair)
{
std::optional<fetchers::Cache::Key> cacheKey;
auto [subpath, fingerprint] =
@ -38,32 +44,47 @@ StorePath fetchToStore(
: path.accessor->getFingerprint(path.path);
if (fingerprint) {
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs());
if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store, mode == FetchMode::DryRun)) {
debug("store path cache hit for '%s'", path);
return res->storePath;
cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath.abs());
if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
debug("source path hash cache hit for '%s'", path);
auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash"));
auto storePath = store.makeFixedOutputPathFromCA(name,
ContentAddressWithReferences::fromParts(method, hash, {}));
if (store.isValidPath(storePath)) {
debug("source path '%s' has valid store path '%s'", path, store.printStorePath(storePath));
return {storePath, hash};
}
debug("source path '%s' not in store", path);
}
} else
debug("source path '%s' is uncacheable (%d, %d)", path, filter, (bool) fingerprint);
// FIXME: could still provide in-memory caching keyed on `SourcePath`.
debug("source path '%s' is uncacheable (%d, %d)", path, (bool) filter, (bool) fingerprint);
Activity act(*logger, lvlChatty, actUnknown,
fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path));
auto filter2 = filter ? *filter : defaultPathFilter;
auto storePath =
mode == FetchMode::DryRun
? store.computeStorePath(
name, path, method, HashAlgorithm::SHA256, {}, filter2).first
: store.addToStore(
if (mode == FetchMode::DryRun) {
auto [storePath, hash] = store.computeStorePath(
name, path, method, HashAlgorithm::SHA256, {}, filter2);
debug("hashed '%s' to '%s'", path, store.printStorePath(storePath));
if (cacheKey)
fetchers::getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}});
return {storePath, hash};
} else {
auto storePath = store.addToStore(
name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
if (cacheKey)
fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);
return storePath;
debug("copied '%s' to '%s'", path, store.printStorePath(storePath));
// FIXME: this is the wrong hash when method !=
// ContentAddressMethod::Raw::NixArchive. Doesn't matter at
// the moment since the only place where that's the case
// doesn't use the hash.
auto hash = store.queryPathInfo(storePath)->narHash;
if (cacheKey)
fetchers::getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}});
return {storePath, hash};
}
}
}