mirror of
https://github.com/NixOS/nix.git
synced 2025-11-08 19:46:02 +01:00
Merge pull request #14050 from NixOS/fix-fetch-to-store-caching
Fix fetchToStore caching
This commit is contained in:
commit
1e709554d5
21 changed files with 188 additions and 92 deletions
|
|
@ -236,24 +236,17 @@ EvalState::EvalState(
|
||||||
{CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
|
{CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
|
||||||
}))
|
}))
|
||||||
, rootFS([&] {
|
, rootFS([&] {
|
||||||
auto accessor = [&]() -> decltype(rootFS) {
|
/* In pure eval mode, we provide a filesystem that only
|
||||||
/* In pure eval mode, we provide a filesystem that only
|
contains the Nix store.
|
||||||
contains the Nix store. */
|
|
||||||
if (settings.pureEval)
|
|
||||||
return storeFS;
|
|
||||||
|
|
||||||
/* If we have a chroot store and pure eval is not enabled,
|
Otherwise, use a union accessor to make the augmented store
|
||||||
use a union accessor to make the chroot store available
|
available at its logical location while still having the
|
||||||
at its logical location while still having the underlying
|
underlying directory available. This is necessary for
|
||||||
directory available. This is necessary for instance if
|
instance if we're evaluating a file from the physical
|
||||||
we're evaluating a file from the physical /nix/store
|
/nix/store while using a chroot store, and also for lazy
|
||||||
while using a chroot store. */
|
mounted fetchTree. */
|
||||||
auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
|
auto accessor = settings.pureEval ? storeFS.cast<SourceAccessor>()
|
||||||
if (store->storeDir != realStoreDir)
|
: makeUnionSourceAccessor({getFSSourceAccessor(), storeFS});
|
||||||
return makeUnionSourceAccessor({getFSSourceAccessor(), storeFS});
|
|
||||||
|
|
||||||
return getFSSourceAccessor();
|
|
||||||
}();
|
|
||||||
|
|
||||||
/* Apply access control if needed. */
|
/* Apply access control if needed. */
|
||||||
if (settings.restrictEval || settings.pureEval)
|
if (settings.restrictEval || settings.pureEval)
|
||||||
|
|
@ -3133,6 +3126,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
|
||||||
auto res = (r / CanonPath(suffix)).resolveSymlinks();
|
auto res = (r / CanonPath(suffix)).resolveSymlinks();
|
||||||
if (res.pathExists())
|
if (res.pathExists())
|
||||||
return res;
|
return res;
|
||||||
|
|
||||||
|
// Backward compatibility hack: throw an exception if access
|
||||||
|
// to this path is not allowed.
|
||||||
|
if (auto accessor = res.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
|
||||||
|
accessor->checkAccess(res.path);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (hasPrefix(path, "nix/"))
|
if (hasPrefix(path, "nix/"))
|
||||||
|
|
@ -3199,6 +3197,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
|
||||||
if (path.resolveSymlinks().pathExists())
|
if (path.resolveSymlinks().pathExists())
|
||||||
return finish(std::move(path));
|
return finish(std::move(path));
|
||||||
else {
|
else {
|
||||||
|
// Backward compatibility hack: throw an exception if access
|
||||||
|
// to this path is not allowed.
|
||||||
|
if (auto accessor = path.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
|
||||||
|
accessor->checkAccess(path.path);
|
||||||
|
|
||||||
logWarning({.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)});
|
logWarning({.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,7 @@ class Store;
|
||||||
namespace fetchers {
|
namespace fetchers {
|
||||||
struct Settings;
|
struct Settings;
|
||||||
struct InputCache;
|
struct InputCache;
|
||||||
|
struct Input;
|
||||||
} // namespace fetchers
|
} // namespace fetchers
|
||||||
struct EvalSettings;
|
struct EvalSettings;
|
||||||
class EvalState;
|
class EvalState;
|
||||||
|
|
@ -575,6 +576,11 @@ public:
|
||||||
|
|
||||||
void checkURI(const std::string & uri);
|
void checkURI(const std::string & uri);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mount an input on the Nix store.
|
||||||
|
*/
|
||||||
|
StorePath mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse a Nix expression from the specified file.
|
* Parse a Nix expression from the specified file.
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
#include "nix/store/store-api.hh"
|
#include "nix/store/store-api.hh"
|
||||||
#include "nix/expr/eval.hh"
|
#include "nix/expr/eval.hh"
|
||||||
|
#include "nix/util/mounted-source-accessor.hh"
|
||||||
|
#include "nix/fetchers/fetch-to-store.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
@ -18,4 +20,27 @@ SourcePath EvalState::storePath(const StorePath & path)
|
||||||
return {rootFS, CanonPath{store->printStorePath(path)}};
|
return {rootFS, CanonPath{store->printStorePath(path)}};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
StorePath
|
||||||
|
EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
|
||||||
|
{
|
||||||
|
auto storePath = fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
|
||||||
|
|
||||||
|
allowPath(storePath); // FIXME: should just whitelist the entire virtual store
|
||||||
|
|
||||||
|
storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
|
||||||
|
|
||||||
|
auto narHash = store->queryPathInfo(storePath)->narHash;
|
||||||
|
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||||
|
|
||||||
|
if (originalInput.getNarHash() && narHash != *originalInput.getNarHash())
|
||||||
|
throw Error(
|
||||||
|
(unsigned int) 102,
|
||||||
|
"NAR hash mismatch in input '%s', expected '%s' but got '%s'",
|
||||||
|
originalInput.to_string(),
|
||||||
|
narHash.to_string(HashFormat::SRI, true),
|
||||||
|
originalInput.getNarHash()->to_string(HashFormat::SRI, true));
|
||||||
|
|
||||||
|
return storePath;
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@
|
||||||
#include "nix/util/url.hh"
|
#include "nix/util/url.hh"
|
||||||
#include "nix/expr/value-to-json.hh"
|
#include "nix/expr/value-to-json.hh"
|
||||||
#include "nix/fetchers/fetch-to-store.hh"
|
#include "nix/fetchers/fetch-to-store.hh"
|
||||||
|
#include "nix/fetchers/input-cache.hh"
|
||||||
|
|
||||||
#include <nlohmann/json.hpp>
|
#include <nlohmann/json.hpp>
|
||||||
|
|
||||||
|
|
@ -218,11 +219,11 @@ static void fetchTree(
|
||||||
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
|
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
auto [storePath, input2] = input.fetchToStore(state.store);
|
auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No);
|
||||||
|
|
||||||
state.allowPath(storePath);
|
auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
|
||||||
|
|
||||||
emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
|
emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
#include "nix/fetchers/fetch-to-store.hh"
|
#include "nix/fetchers/fetch-to-store.hh"
|
||||||
#include "nix/fetchers/fetchers.hh"
|
#include "nix/fetchers/fetchers.hh"
|
||||||
#include "nix/fetchers/fetch-settings.hh"
|
#include "nix/fetchers/fetch-settings.hh"
|
||||||
|
#include "nix/util/environment-variables.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
@ -27,14 +28,22 @@ StorePath fetchToStore(
|
||||||
|
|
||||||
std::optional<fetchers::Cache::Key> cacheKey;
|
std::optional<fetchers::Cache::Key> cacheKey;
|
||||||
|
|
||||||
if (!filter && path.accessor->fingerprint) {
|
auto [subpath, fingerprint] = filter ? std::pair<CanonPath, std::optional<std::string>>{path.path, std::nullopt}
|
||||||
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
|
: path.accessor->getFingerprint(path.path);
|
||||||
|
|
||||||
|
if (fingerprint) {
|
||||||
|
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs());
|
||||||
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
|
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
|
||||||
debug("store path cache hit for '%s'", path);
|
debug("store path cache hit for '%s'", path);
|
||||||
return res->storePath;
|
return res->storePath;
|
||||||
}
|
}
|
||||||
} else
|
} else {
|
||||||
|
static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
|
||||||
|
if (barf && !filter)
|
||||||
|
throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter);
|
||||||
|
// FIXME: could still provide in-memory caching keyed on `SourcePath`.
|
||||||
debug("source path '%s' is uncacheable", path);
|
debug("source path '%s' is uncacheable", path);
|
||||||
|
}
|
||||||
|
|
||||||
Activity act(
|
Activity act(
|
||||||
*logger,
|
*logger,
|
||||||
|
|
|
||||||
|
|
@ -356,8 +356,10 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
|
||||||
|
|
||||||
auto [accessor, result] = scheme->getAccessor(store, *this);
|
auto [accessor, result] = scheme->getAccessor(store, *this);
|
||||||
|
|
||||||
assert(!accessor->fingerprint);
|
if (!accessor->fingerprint)
|
||||||
accessor->fingerprint = result.getFingerprint(store);
|
accessor->fingerprint = result.getFingerprint(store);
|
||||||
|
else
|
||||||
|
result.cachedFingerprint = accessor->fingerprint;
|
||||||
|
|
||||||
return {accessor, std::move(result)};
|
return {accessor, std::move(result)};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -16,15 +16,26 @@ std::string FilteringSourceAccessor::readFile(const CanonPath & path)
|
||||||
return next->readFile(prefix / path);
|
return next->readFile(prefix / path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
|
||||||
|
{
|
||||||
|
checkAccess(path);
|
||||||
|
return next->readFile(prefix / path, sink, sizeCallback);
|
||||||
|
}
|
||||||
|
|
||||||
bool FilteringSourceAccessor::pathExists(const CanonPath & path)
|
bool FilteringSourceAccessor::pathExists(const CanonPath & path)
|
||||||
{
|
{
|
||||||
return isAllowed(path) && next->pathExists(prefix / path);
|
return isAllowed(path) && next->pathExists(prefix / path);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<SourceAccessor::Stat> FilteringSourceAccessor::maybeLstat(const CanonPath & path)
|
std::optional<SourceAccessor::Stat> FilteringSourceAccessor::maybeLstat(const CanonPath & path)
|
||||||
|
{
|
||||||
|
return isAllowed(path) ? next->maybeLstat(prefix / path) : std::nullopt;
|
||||||
|
}
|
||||||
|
|
||||||
|
SourceAccessor::Stat FilteringSourceAccessor::lstat(const CanonPath & path)
|
||||||
{
|
{
|
||||||
checkAccess(path);
|
checkAccess(path);
|
||||||
return next->maybeLstat(prefix / path);
|
return next->lstat(prefix / path);
|
||||||
}
|
}
|
||||||
|
|
||||||
SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path)
|
SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path)
|
||||||
|
|
@ -49,6 +60,13 @@ std::string FilteringSourceAccessor::showPath(const CanonPath & path)
|
||||||
return displayPrefix + next->showPath(prefix / path) + displaySuffix;
|
return displayPrefix + next->showPath(prefix / path) + displaySuffix;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFingerprint(const CanonPath & path)
|
||||||
|
{
|
||||||
|
if (fingerprint)
|
||||||
|
return {path, fingerprint};
|
||||||
|
return next->getFingerprint(prefix / path);
|
||||||
|
}
|
||||||
|
|
||||||
void FilteringSourceAccessor::checkAccess(const CanonPath & path)
|
void FilteringSourceAccessor::checkAccess(const CanonPath & path)
|
||||||
{
|
{
|
||||||
if (!isAllowed(path))
|
if (!isAllowed(path))
|
||||||
|
|
|
||||||
|
|
@ -893,8 +893,7 @@ struct GitInputScheme : InputScheme
|
||||||
return makeFingerprint(*rev);
|
return makeFingerprint(*rev);
|
||||||
else {
|
else {
|
||||||
auto repoInfo = getRepoInfo(input);
|
auto repoInfo = getRepoInfo(input);
|
||||||
if (auto repoPath = repoInfo.getPath();
|
if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) {
|
||||||
repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) {
|
|
||||||
/* Calculate a fingerprint that takes into account the
|
/* Calculate a fingerprint that takes into account the
|
||||||
deleted and modified/added files. */
|
deleted and modified/added files. */
|
||||||
HashSink hashSink{HashAlgorithm::SHA512};
|
HashSink hashSink{HashAlgorithm::SHA512};
|
||||||
|
|
@ -907,7 +906,7 @@ struct GitInputScheme : InputScheme
|
||||||
writeString("deleted:", hashSink);
|
writeString("deleted:", hashSink);
|
||||||
writeString(file.abs(), hashSink);
|
writeString(file.abs(), hashSink);
|
||||||
}
|
}
|
||||||
return makeFingerprint(*repoInfo.workdirInfo.headRev)
|
return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev))
|
||||||
+ ";d=" + hashSink.finish().hash.to_string(HashFormat::Base16, false);
|
+ ";d=" + hashSink.finish().hash.to_string(HashFormat::Base16, false);
|
||||||
}
|
}
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
|
|
|
||||||
|
|
@ -36,8 +36,12 @@ struct FilteringSourceAccessor : SourceAccessor
|
||||||
|
|
||||||
std::string readFile(const CanonPath & path) override;
|
std::string readFile(const CanonPath & path) override;
|
||||||
|
|
||||||
|
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
|
||||||
|
|
||||||
bool pathExists(const CanonPath & path) override;
|
bool pathExists(const CanonPath & path) override;
|
||||||
|
|
||||||
|
Stat lstat(const CanonPath & path) override;
|
||||||
|
|
||||||
std::optional<Stat> maybeLstat(const CanonPath & path) override;
|
std::optional<Stat> maybeLstat(const CanonPath & path) override;
|
||||||
|
|
||||||
DirEntries readDirectory(const CanonPath & path) override;
|
DirEntries readDirectory(const CanonPath & path) override;
|
||||||
|
|
@ -46,6 +50,8 @@ struct FilteringSourceAccessor : SourceAccessor
|
||||||
|
|
||||||
std::string showPath(const CanonPath & path) override;
|
std::string showPath(const CanonPath & path) override;
|
||||||
|
|
||||||
|
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Call `makeNotAllowedError` to throw a `RestrictedPathError`
|
* Call `makeNotAllowedError` to throw a `RestrictedPathError`
|
||||||
* exception if `isAllowed()` returns `false` for `path`.
|
* exception if `isAllowed()` returns `false` for `path`.
|
||||||
|
|
|
||||||
|
|
@ -123,8 +123,6 @@ struct PathInputScheme : InputScheme
|
||||||
|
|
||||||
auto absPath = getAbsPath(input);
|
auto absPath = getAbsPath(input);
|
||||||
|
|
||||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
|
|
||||||
|
|
||||||
// FIXME: check whether access to 'path' is allowed.
|
// FIXME: check whether access to 'path' is allowed.
|
||||||
auto storePath = store->maybeParseStorePath(absPath.string());
|
auto storePath = store->maybeParseStorePath(absPath.string());
|
||||||
|
|
||||||
|
|
@ -133,43 +131,33 @@ struct PathInputScheme : InputScheme
|
||||||
|
|
||||||
time_t mtime = 0;
|
time_t mtime = 0;
|
||||||
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
|
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
|
||||||
|
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
|
||||||
// FIXME: try to substitute storePath.
|
// FIXME: try to substitute storePath.
|
||||||
auto src = sinkToSource(
|
auto src = sinkToSource(
|
||||||
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
|
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
|
||||||
storePath = store->addToStoreFromDump(*src, "source");
|
storePath = store->addToStoreFromDump(*src, "source");
|
||||||
}
|
}
|
||||||
|
|
||||||
// To avoid copying the path again to the /nix/store, we need to add a cache entry.
|
auto accessor = ref{store->getFSAccessor(*storePath)};
|
||||||
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
|
|
||||||
auto fp = getFingerprint(store, input);
|
// To prevent `fetchToStore()` copying the path again to Nix
|
||||||
if (fp) {
|
// store, pre-create an entry in the fetcher cache.
|
||||||
auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/");
|
auto info = store->queryPathInfo(*storePath);
|
||||||
input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath);
|
accessor->fingerprint =
|
||||||
}
|
fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
|
||||||
|
input.settings->getCache()->upsert(
|
||||||
|
makeFetchToStoreCacheKey(
|
||||||
|
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
|
||||||
|
*store,
|
||||||
|
{},
|
||||||
|
*storePath);
|
||||||
|
|
||||||
/* Trust the lastModified value supplied by the user, if
|
/* Trust the lastModified value supplied by the user, if
|
||||||
any. It's not a "secure" attribute so we don't care. */
|
any. It's not a "secure" attribute so we don't care. */
|
||||||
if (!input.getLastModified())
|
if (!input.getLastModified())
|
||||||
input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
|
input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
|
||||||
|
|
||||||
return {ref{store->getFSAccessor(*storePath)}, std::move(input)};
|
return {accessor, std::move(input)};
|
||||||
}
|
|
||||||
|
|
||||||
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
|
|
||||||
{
|
|
||||||
if (isRelative(input))
|
|
||||||
return std::nullopt;
|
|
||||||
|
|
||||||
/* If this path is in the Nix store, use the hash of the
|
|
||||||
store object and the subpath. */
|
|
||||||
auto path = getAbsPath(input);
|
|
||||||
try {
|
|
||||||
auto [storePath, subPath] = store->toStorePath(path.string());
|
|
||||||
auto info = store->queryPathInfo(storePath);
|
|
||||||
return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath);
|
|
||||||
} catch (Error &) {
|
|
||||||
return std::nullopt;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<ExperimentalFeature> experimentalFeature() const override
|
std::optional<ExperimentalFeature> experimentalFeature() const override
|
||||||
|
|
|
||||||
|
|
@ -24,21 +24,6 @@ using namespace flake;
|
||||||
|
|
||||||
namespace flake {
|
namespace flake {
|
||||||
|
|
||||||
static StorePath copyInputToStore(
|
|
||||||
EvalState & state, fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
|
|
||||||
{
|
|
||||||
auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName());
|
|
||||||
|
|
||||||
state.allowPath(storePath);
|
|
||||||
|
|
||||||
auto narHash = state.store->queryPathInfo(storePath)->narHash;
|
|
||||||
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
|
||||||
|
|
||||||
assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
|
|
||||||
|
|
||||||
return storePath;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
|
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
|
||||||
{
|
{
|
||||||
if (value.isThunk() && value.isTrivial())
|
if (value.isThunk() && value.isTrivial())
|
||||||
|
|
@ -360,11 +345,14 @@ static Flake getFlake(
|
||||||
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
|
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy the tree to the store.
|
|
||||||
auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, cachedInput.accessor);
|
|
||||||
|
|
||||||
// Re-parse flake.nix from the store.
|
// Re-parse flake.nix from the store.
|
||||||
return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath);
|
return readFlake(
|
||||||
|
state,
|
||||||
|
originalRef,
|
||||||
|
resolvedRef,
|
||||||
|
lockedRef,
|
||||||
|
state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)),
|
||||||
|
lockRootAttrPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries)
|
Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries)
|
||||||
|
|
@ -721,11 +709,10 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
|
||||||
|
|
||||||
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
|
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
|
||||||
|
|
||||||
// FIXME: allow input to be lazy.
|
return {
|
||||||
auto storePath = copyInputToStore(
|
state.storePath(
|
||||||
state, lockedRef.input, input.ref->input, cachedInput.accessor);
|
state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor)),
|
||||||
|
lockedRef};
|
||||||
return {state.storePath(storePath), lockedRef};
|
|
||||||
}
|
}
|
||||||
}();
|
}();
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -121,7 +121,7 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
|
||||||
std::string typeString();
|
std::string typeString();
|
||||||
};
|
};
|
||||||
|
|
||||||
Stat lstat(const CanonPath & path);
|
virtual Stat lstat(const CanonPath & path);
|
||||||
|
|
||||||
virtual std::optional<Stat> maybeLstat(const CanonPath & path) = 0;
|
virtual std::optional<Stat> maybeLstat(const CanonPath & path) = 0;
|
||||||
|
|
||||||
|
|
@ -180,6 +180,27 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
|
||||||
*/
|
*/
|
||||||
std::optional<std::string> fingerprint;
|
std::optional<std::string> fingerprint;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the fingerprint for `path`. This is usually the
|
||||||
|
* fingerprint of the current accessor, but for composite
|
||||||
|
* accessors (like `MountedSourceAccessor`), we want to return the
|
||||||
|
* fingerprint of the "inner" accessor if the current one lacks a
|
||||||
|
* fingerprint.
|
||||||
|
*
|
||||||
|
* So this method is intended to return the most-outer accessor
|
||||||
|
* that has a fingerprint for `path`. It also returns the path that `path`
|
||||||
|
* corresponds to in that accessor.
|
||||||
|
*
|
||||||
|
* For example: in a `MountedSourceAccessor` that has
|
||||||
|
* `/nix/store/foo` mounted,
|
||||||
|
* `getFingerprint("/nix/store/foo/bar")` will return the path
|
||||||
|
* `/bar` and the fingerprint of the `/nix/store/foo` accessor.
|
||||||
|
*/
|
||||||
|
virtual std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path)
|
||||||
|
{
|
||||||
|
return {path, fingerprint};
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return the maximum last-modified time of the files in this
|
* Return the maximum last-modified time of the files in this
|
||||||
* tree, if available.
|
* tree, if available.
|
||||||
|
|
|
||||||
|
|
@ -27,6 +27,12 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor
|
||||||
return accessor->readFile(subpath);
|
return accessor->readFile(subpath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Stat lstat(const CanonPath & path) override
|
||||||
|
{
|
||||||
|
auto [accessor, subpath] = resolve(path);
|
||||||
|
return accessor->lstat(subpath);
|
||||||
|
}
|
||||||
|
|
||||||
std::optional<Stat> maybeLstat(const CanonPath & path) override
|
std::optional<Stat> maybeLstat(const CanonPath & path) override
|
||||||
{
|
{
|
||||||
auto [accessor, subpath] = resolve(path);
|
auto [accessor, subpath] = resolve(path);
|
||||||
|
|
@ -85,6 +91,14 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor
|
||||||
else
|
else
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override
|
||||||
|
{
|
||||||
|
if (fingerprint)
|
||||||
|
return {path, fingerprint};
|
||||||
|
auto [accessor, subpath] = resolve(path);
|
||||||
|
return accessor->getFingerprint(subpath);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts)
|
ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts)
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,18 @@ struct UnionSourceAccessor : SourceAccessor
|
||||||
}
|
}
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override
|
||||||
|
{
|
||||||
|
if (fingerprint)
|
||||||
|
return {path, fingerprint};
|
||||||
|
for (auto & accessor : accessors) {
|
||||||
|
auto [subpath, fingerprint] = accessor->getFingerprint(path);
|
||||||
|
if (fingerprint)
|
||||||
|
return {subpath, fingerprint};
|
||||||
|
}
|
||||||
|
return {path, std::nullopt};
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<SourceAccessor> makeUnionSourceAccessor(std::vector<ref<SourceAccessor>> && accessors)
|
ref<SourceAccessor> makeUnionSourceAccessor(std::vector<ref<SourceAccessor>> && accessors)
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,8 @@
|
||||||
|
|
||||||
source ../common.sh
|
source ../common.sh
|
||||||
|
|
||||||
|
export _NIX_TEST_BARF_ON_UNCACHEABLE=1
|
||||||
|
|
||||||
# shellcheck disable=SC2034 # this variable is used by tests that source this file
|
# shellcheck disable=SC2034 # this variable is used by tests that source this file
|
||||||
registry=$TEST_ROOT/registry.json
|
registry=$TEST_ROOT/registry.json
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -62,8 +62,8 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule
|
||||||
# Check that dirtying a submodule makes the entire thing dirty.
|
# Check that dirtying a submodule makes the entire thing dirty.
|
||||||
[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]]
|
[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]]
|
||||||
echo '"foo"' > "$rootRepo"/submodule/sub.nix
|
echo '"foo"' > "$rootRepo"/submodule/sub.nix
|
||||||
[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]]
|
[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval --json "$flakeref#sub" ) = '"foo"' ]]
|
||||||
[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]]
|
[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]]
|
||||||
|
|
||||||
# Test that `nix flake metadata` parses `submodule` correctly.
|
# Test that `nix flake metadata` parses `submodule` correctly.
|
||||||
cat > "$rootRepo"/flake.nix <<EOF
|
cat > "$rootRepo"/flake.nix <<EOF
|
||||||
|
|
@ -75,7 +75,7 @@ EOF
|
||||||
git -C "$rootRepo" add flake.nix
|
git -C "$rootRepo" add flake.nix
|
||||||
git -C "$rootRepo" commit -m "Add flake.nix"
|
git -C "$rootRepo" commit -m "Add flake.nix"
|
||||||
|
|
||||||
storePath=$(nix flake prefetch --json "$rootRepo?submodules=1" | jq -r .storePath)
|
storePath=$(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake prefetch --json "$rootRepo?submodules=1" | jq -r .storePath)
|
||||||
[[ -e "$storePath/submodule" ]]
|
[[ -e "$storePath/submodule" ]]
|
||||||
|
|
||||||
# Test the use of inputs.self.
|
# Test the use of inputs.self.
|
||||||
|
|
|
||||||
|
|
@ -131,7 +131,7 @@ EOF
|
||||||
git -C "$flakeFollowsA" add flake.nix
|
git -C "$flakeFollowsA" add flake.nix
|
||||||
|
|
||||||
expect 1 nix flake lock "$flakeFollowsA" 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode'
|
expect 1 nix flake lock "$flakeFollowsA" 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode'
|
||||||
expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep '/flakeB.*does not exist'
|
expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep "'flakeB' is too short to be a valid store path"
|
||||||
|
|
||||||
# Test relative non-flake inputs.
|
# Test relative non-flake inputs.
|
||||||
cat > "$flakeFollowsA"/flake.nix <<EOF
|
cat > "$flakeFollowsA"/flake.nix <<EOF
|
||||||
|
|
|
||||||
|
|
@ -27,9 +27,9 @@ nix build -o "$TEST_ROOT/result" "hg+file://$flake2Dir"
|
||||||
|
|
||||||
(! nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revision)
|
(! nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revision)
|
||||||
|
|
||||||
nix eval "hg+file://$flake2Dir"#expr
|
_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr
|
||||||
|
|
||||||
nix eval "hg+file://$flake2Dir"#expr
|
_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr
|
||||||
|
|
||||||
(! nix eval "hg+file://$flake2Dir"#expr --no-allow-dirty)
|
(! nix eval "hg+file://$flake2Dir"#expr --no-allow-dirty)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -81,7 +81,7 @@ nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file
|
||||||
|
|
||||||
nix registry add --registry "$registry" flake3 "git+file://$flake3Dir"
|
nix registry add --registry "$registry" flake3 "git+file://$flake3Dir"
|
||||||
|
|
||||||
nix build -o "$TEST_ROOT/result" flake3#fnord
|
_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord
|
||||||
[[ $(cat "$TEST_ROOT/result") = FNORD ]]
|
[[ $(cat "$TEST_ROOT/result") = FNORD ]]
|
||||||
|
|
||||||
# Check whether flake input fetching is lazy: flake3#sth does not
|
# Check whether flake input fetching is lazy: flake3#sth does not
|
||||||
|
|
@ -91,16 +91,17 @@ clearStore
|
||||||
mv "$flake2Dir" "$flake2Dir.tmp"
|
mv "$flake2Dir" "$flake2Dir.tmp"
|
||||||
mv "$nonFlakeDir" "$nonFlakeDir.tmp"
|
mv "$nonFlakeDir" "$nonFlakeDir.tmp"
|
||||||
nix build -o "$TEST_ROOT/result" flake3#sth
|
nix build -o "$TEST_ROOT/result" flake3#sth
|
||||||
(! nix build -o "$TEST_ROOT/result" flake3#xyzzy)
|
(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy)
|
||||||
(! nix build -o "$TEST_ROOT/result" flake3#fnord)
|
(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord)
|
||||||
mv "$flake2Dir.tmp" "$flake2Dir"
|
mv "$flake2Dir.tmp" "$flake2Dir"
|
||||||
mv "$nonFlakeDir.tmp" "$nonFlakeDir"
|
mv "$nonFlakeDir.tmp" "$nonFlakeDir"
|
||||||
nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord
|
_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord
|
||||||
|
|
||||||
# Check non-flake inputs have a sourceInfo and an outPath
|
# Check non-flake inputs have a sourceInfo and an outPath
|
||||||
#
|
#
|
||||||
# This may look redundant, but the other checks below happen in a command
|
# This may look redundant, but the other checks below happen in a command
|
||||||
# substitution subshell, so failures there will not exit this shell
|
# substitution subshell, so failures there will not exit this shell
|
||||||
|
export _NIX_TEST_BARF_ON_UNCACHEABLE='' # FIXME
|
||||||
nix eval --raw flake3#inputs.nonFlake.outPath
|
nix eval --raw flake3#inputs.nonFlake.outPath
|
||||||
nix eval --raw flake3#inputs.nonFlake.sourceInfo.outPath
|
nix eval --raw flake3#inputs.nonFlake.sourceInfo.outPath
|
||||||
nix eval --raw flake3#inputs.nonFlakeFile.outPath
|
nix eval --raw flake3#inputs.nonFlakeFile.outPath
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@ source ./common.sh
|
||||||
|
|
||||||
requireGit
|
requireGit
|
||||||
|
|
||||||
|
unset _NIX_TEST_BARF_ON_UNCACHEABLE
|
||||||
|
|
||||||
# Test a "vendored" subflake dependency. This is a relative path flake
|
# Test a "vendored" subflake dependency. This is a relative path flake
|
||||||
# which doesn't reference the root flake and has its own lock file.
|
# which doesn't reference the root flake and has its own lock file.
|
||||||
#
|
#
|
||||||
|
|
|
||||||
|
|
@ -10,4 +10,4 @@ error:
|
||||||
|
|
||||||
… while calling the 'hashFile' builtin
|
… while calling the 'hashFile' builtin
|
||||||
|
|
||||||
error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory
|
error: path '/pwd/lang/this-file-is-definitely-not-there-7392097' does not exist
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue