nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-15 05:21:03 +01:00
Code Activity

Merge remote-tracking branch 'upstream/master' into upstream-RossComputerGuy/feat/expose-computefsclosure

Browse source
This commit is contained in:
Robert Hensing 2025-10-15 15:30:43 +02:00
commit a9d9b50b72
467 changed files with 9259 additions and 5039 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/libcmd/built-path.cc
View file

@ -83,12 +83,22 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) cons
nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const
{
return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
return std::visit(
overloaded{
[&](const SingleBuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); },
[&](const SingleBuiltPath::Built & b) { return b.toJSON(store); },
},
raw());
}
nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const
{
return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
return std::visit(
overloaded{
[&](const BuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); },
[&](const BuiltPath::Built & b) { return b.toJSON(store); },
},
raw());
}
RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const

4
src/libcmd/include/nix/cmd/installable-flake.hh
View file

@ -69,7 +69,7 @@ struct InstallableFlake : InstallableValue
*/
std::vector<ref<eval_cache::AttrCursor>> getCursors(EvalState & state) override;
std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
ref<flake::LockedFlake> getLockedFlake() const;
FlakeRef nixpkgsFlakeRef() const;
};
@ -87,6 +87,4 @@ static inline FlakeRef defaultNixpkgsFlakeRef()
return FlakeRef::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", "nixpkgs"}});
}
ref<eval_cache::EvalCache> openEvalCache(EvalState & state, std::shared_ptr<flake::LockedFlake> lockedFlake);
} // namespace nix

8
src/libcmd/installable-flake.cc
View file

@ -185,16 +185,16 @@ std::vector<ref<eval_cache::AttrCursor>> InstallableFlake::getCursors(EvalState
return res;
}
std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
ref<flake::LockedFlake> InstallableFlake::getLockedFlake() const
{
if (!_lockedFlake) {
flake::LockFlags lockFlagsApplyConfig = lockFlags;
// FIXME why this side effect?
lockFlagsApplyConfig.applyNixConfig = true;
_lockedFlake =
std::make_shared<flake::LockedFlake>(lockFlake(flakeSettings, *state, flakeRef, lockFlagsApplyConfig));
_lockedFlake = make_ref<flake::LockedFlake>(lockFlake(flakeSettings, *state, flakeRef, lockFlagsApplyConfig));
}
return _lockedFlake;
// _lockedFlake is now non-null but still just a shared_ptr
return ref<flake::LockedFlake>(_lockedFlake);
}
FlakeRef InstallableFlake::nixpkgsFlakeRef() const

57
src/libcmd/installables.cc
View file

@ -342,8 +342,7 @@ void completeFlakeRefWithFragment(
parseFlakeRef(fetchSettings, expandTilde(flakeRefS), std::filesystem::current_path().string());
auto evalCache = openEvalCache(
*evalState,
std::make_shared<flake::LockedFlake>(lockFlake(flakeSettings, *evalState, flakeRef, lockFlags)));
*evalState, make_ref<flake::LockedFlake>(lockFlake(flakeSettings, *evalState, flakeRef, lockFlags)));
auto root = evalCache->getRoot();
@ -443,42 +442,6 @@ static StorePath getDeriver(ref<Store> store, const Installable & i, const Store
return *derivers.begin();
}
ref<eval_cache::EvalCache> openEvalCache(EvalState & state, std::shared_ptr<flake::LockedFlake> lockedFlake)
{
auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval
? lockedFlake->getFingerprint(state.store, state.fetchSettings)
: std::nullopt;
auto rootLoader = [&state, lockedFlake]() {
/* For testing whether the evaluation cache is
complete. */
if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
throw Error("not everything is cached, but evaluation is not allowed");
auto vFlake = state.allocValue();
flake::callFlake(state, *lockedFlake, *vFlake);
state.forceAttrs(*vFlake, noPos, "while parsing cached flake data");
auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs"));
assert(aOutputs);
return aOutputs->value;
};
if (fingerprint) {
auto search = state.evalCaches.find(fingerprint.value());
if (search == state.evalCaches.end()) {
search =
state.evalCaches
.emplace(fingerprint.value(), make_ref<nix::eval_cache::EvalCache>(fingerprint, state, rootLoader))
.first;
}
return search->second;
} else {
return make_ref<nix::eval_cache::EvalCache>(std::nullopt, state, rootLoader);
}
}
Installables SourceExprCommand::parseInstallables(ref<Store> store, std::vector<std::string> ss)
{
Installables result;
@ -604,28 +567,28 @@ std::vector<BuiltPathWithResult> Installable::build(
static void throwBuildErrors(std::vector<KeyedBuildResult> & buildResults, const Store & store)
{
std::vector<KeyedBuildResult> failed;
std::vector<std::pair<const KeyedBuildResult *, const KeyedBuildResult::Failure *>> failed;
for (auto & buildResult : buildResults) {
if (!buildResult.success()) {
failed.push_back(buildResult);
if (auto * failure = buildResult.tryGetFailure()) {
failed.push_back({&buildResult, failure});
}
}
auto failedResult = failed.begin();
if (failedResult != failed.end()) {
if (failed.size() == 1) {
failedResult->rethrow();
failedResult->second->rethrow();
} else {
StringSet failedPaths;
for (; failedResult != failed.end(); failedResult++) {
if (!failedResult->errorMsg.empty()) {
if (!failedResult->second->errorMsg.empty()) {
logError(
ErrorInfo{
.level = lvlError,
.msg = failedResult->errorMsg,
.msg = failedResult->second->errorMsg,
});
}
failedPaths.insert(failedResult->path.to_string(store));
failedPaths.insert(failedResult->first->path.to_string(store));
}
throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths)));
}
@ -695,12 +658,14 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore);
throwBuildErrors(buildResults, *store);
for (auto & buildResult : buildResults) {
// If we didn't throw, they must all be sucesses
auto & success = std::get<nix::BuildResult::Success>(buildResult.inner);
for (auto & aux : backmap[buildResult.path]) {
std::visit(
overloaded{
[&](const DerivedPath::Built & bfd) {
std::map<std::string, StorePath> outputs;
for (auto & [outputName, realisation] : buildResult.builtOutputs)
for (auto & [outputName, realisation] : success.builtOutputs)
outputs.emplace(outputName, realisation.outPath);
res.push_back(
{aux.installable,

1
src/libcmd/meson.build
View file

@ -67,7 +67,6 @@ config_priv_h = configure_file(
)
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'built-path.cc',

17
src/libcmd/repl.cc
View file

@ -669,7 +669,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
ss << "No documentation found.\n\n";
}
auto markdown = toView(ss);
auto markdown = ss.view();
logger->cout(trim(renderMarkdownToTerminal(markdown)));
} else
@ -760,7 +760,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
void NixRepl::initEnv()
{
env = &state->allocEnv(envSize);
env = &state->mem.allocEnv(envSize);
env->up = &state->baseEnv;
displ = 0;
staticEnv->vars.clear();
@ -869,14 +869,8 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
Expr * NixRepl::parseString(std::string s)
{
return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv);
}
void NixRepl::evalString(std::string s, Value & v)
{
Expr * e;
try {
e = parseString(s);
return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv);
} catch (ParseError & e) {
if (e.msg().find("unexpected end of file") != std::string::npos)
// For parse errors on incomplete input, we continue waiting for the next line of
@ -885,6 +879,11 @@ void NixRepl::evalString(std::string s, Value & v)
else
throw;
}
}
void NixRepl::evalString(std::string s, Value & v)
{
Expr * e = parseString(s);
e->eval(*state, *env, v);
state->forceValue(v, v.determinePos(noPos));
}

1
src/libexpr-c/meson.build
View file

@ -28,7 +28,6 @@ deps_public_maybe_subproject = [
subdir('nix-meson-build-support/subprojects')
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'nix_api_expr.cc',

76
src/libexpr-c/nix_api_value.cc
View file

@ -326,6 +326,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
try {
auto & v = check_value_in(value);
assert(v.type() == nix::nList);
if (ix >= v.listSize()) {
nix_set_err_msg(context, NIX_ERR_KEY, "list index out of bounds");
return nullptr;
}
auto * p = v.listView()[ix];
nix_gc_incref(nullptr, p);
if (p != nullptr)
@ -335,6 +339,26 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
NIXC_CATCH_ERRS_NULL
}
nix_value *
nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix)
{
if (context)
context->last_err_code = NIX_OK;
try {
auto & v = check_value_in(value);
assert(v.type() == nix::nList);
if (ix >= v.listSize()) {
nix_set_err_msg(context, NIX_ERR_KEY, "list index out of bounds");
return nullptr;
}
auto * p = v.listView()[ix];
nix_gc_incref(nullptr, p);
// Note: intentionally NOT calling forceValue() to keep the element lazy
return as_nix_value_ptr(p);
}
NIXC_CATCH_ERRS_NULL
}
nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
{
if (context)
@ -355,6 +379,27 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
NIXC_CATCH_ERRS_NULL
}
nix_value *
nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
{
if (context)
context->last_err_code = NIX_OK;
try {
auto & v = check_value_in(value);
assert(v.type() == nix::nAttrs);
nix::Symbol s = state->state.symbols.create(name);
auto attr = v.attrs()->get(s);
if (attr) {
nix_gc_incref(nullptr, attr->value);
// Note: intentionally NOT calling forceValue() to keep the attribute lazy
return as_nix_value_ptr(attr->value);
}
nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
return nullptr;
}
NIXC_CATCH_ERRS_NULL
}
bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name)
{
if (context)
@ -389,6 +434,10 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state
try {
auto & v = check_value_in(value);
collapse_attrset_layer_chain_if_needed(v, state);
if (i >= v.attrs()->size()) {
nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds");
return nullptr;
}
const nix::Attr & a = (*v.attrs())[i];
*name = state->state.symbols[a.name].c_str();
nix_gc_incref(nullptr, a.value);
@ -398,6 +447,27 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state
NIXC_CATCH_ERRS_NULL
}
nix_value * nix_get_attr_byidx_lazy(
nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name)
{
if (context)
context->last_err_code = NIX_OK;
try {
auto & v = check_value_in(value);
collapse_attrset_layer_chain_if_needed(v, state);
if (i >= v.attrs()->size()) {
nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds (Nix C API contract violation)");
return nullptr;
}
const nix::Attr & a = (*v.attrs())[i];
*name = state->state.symbols[a.name].c_str();
nix_gc_incref(nullptr, a.value);
// Note: intentionally NOT calling forceValue() to keep the attribute lazy
return as_nix_value_ptr(a.value);
}
NIXC_CATCH_ERRS_NULL
}
const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i)
{
if (context)
@ -405,6 +475,10 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value,
try {
auto & v = check_value_in(value);
collapse_attrset_layer_chain_if_needed(v, state);
if (i >= v.attrs()->size()) {
nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds (Nix C API contract violation)");
return nullptr;
}
const nix::Attr & a = (*v.attrs())[i];
return state->state.symbols[a.name].c_str();
}
@ -605,7 +679,7 @@ nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * b
context->last_err_code = NIX_OK;
try {
auto & v = check_value_not_null(value);
nix::Symbol s = bb->builder.state.get().symbols.create(name);
nix::Symbol s = bb->builder.symbols.get().create(name);
bb->builder.insert(s, &v);
}
NIXC_CATCH_ERRS

88
src/libexpr-c/nix_api_value.h
View file

@ -265,10 +265,25 @@ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value);
*/
nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
/** @brief Get an attr by name
/** @brief Get the ix'th element of a list without forcing evaluation of the element
*
* Returns the list element without forcing its evaluation, allowing access to lazy values.
* The list value itself must already be evaluated.
*
* Owned by the GC. Use nix_gc_decref when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect (must be an evaluated list)
* @param[in] state nix evaluator state
* @param[in] ix list element to get
* @return value, NULL in case of errors
*/
nix_value *
nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
/** @brief Get an attr by name
*
* Use nix_gc_decref when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @param[in] state nix evaluator state
* @param[in] name attribute name
@ -276,6 +291,21 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
*/
nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
/** @brief Get an attribute value by attribute name, without forcing evaluation of the attribute's value
*
* Returns the attribute value without forcing its evaluation, allowing access to lazy values.
* The attribute set value itself must already be evaluated.
*
* Use nix_gc_decref when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect (must be an evaluated attribute set)
* @param[in] state nix evaluator state
* @param[in] name attribute name
* @return value, NULL in case of errors
*/
nix_value *
nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
/** @brief Check if an attribute name exists on a value
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
@ -285,11 +315,21 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
*/
bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
/** @brief Get an attribute by index in the sorted bindings
/** @brief Get an attribute by index
*
* Also gives you the name.
*
* Owned by the GC. Use nix_gc_decref when you're done with the pointer
* Attributes are returned in an unspecified order which is NOT suitable for
* reproducible operations. In Nix's domain, reproducibility is paramount. The caller
* is responsible for sorting the attributes or storing them in an ordered map to
* ensure deterministic behavior in your application.
*
* @note When Nix does sort attributes, which it does for virtually all intermediate
* operations and outputs, it uses byte-wise lexicographic order (equivalent to
* lexicographic order by Unicode scalar value for valid UTF-8). We recommend
* applying this same ordering for consistency.
*
* Use nix_gc_decref when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @param[in] state nix evaluator state
@ -300,9 +340,47 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS
nix_value *
nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name);
/** @brief Get an attribute name by index in the sorted bindings
/** @brief Get an attribute by index, without forcing evaluation of the attribute's value
*
* Useful when you want the name but want to avoid evaluation.
* Also gives you the name.
*
* Returns the attribute value without forcing its evaluation, allowing access to lazy values.
* The attribute set value itself must already have been evaluated.
*
* Attributes are returned in an unspecified order which is NOT suitable for
* reproducible operations. In Nix's domain, reproducibility is paramount. The caller
* is responsible for sorting the attributes or storing them in an ordered map to
* ensure deterministic behavior in your application.
*
* @note When Nix does sort attributes, which it does for virtually all intermediate
* operations and outputs, it uses byte-wise lexicographic order (equivalent to
* lexicographic order by Unicode scalar value for valid UTF-8). We recommend
* applying this same ordering for consistency.
*
* Use nix_gc_decref when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect (must be an evaluated attribute set)
* @param[in] state nix evaluator state
* @param[in] i attribute index
* @param[out] name will store a pointer to the attribute name
* @return value, NULL in case of errors
*/
nix_value * nix_get_attr_byidx_lazy(
nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name);
/** @brief Get an attribute name by index
*
* Returns the attribute name without forcing evaluation of the attribute's value.
*
* Attributes are returned in an unspecified order which is NOT suitable for
* reproducible operations. In Nix's domain, reproducibility is paramount. The caller
* is responsible for sorting the attributes or storing them in an ordered map to
* ensure deterministic behavior in your application.
*
* @note When Nix does sort attributes, which it does for virtually all intermediate
* operations and outputs, it uses byte-wise lexicographic order (equivalent to
* lexicographic order by Unicode scalar value for valid UTF-8). We recommend
* applying this same ordering for consistency.
*
* Owned by the nix EvalState
* @param[out] context Optional, stores error information

13
src/libexpr-test-support/include/nix/expr/tests/libexpr.hh
View file

@ -26,11 +26,20 @@ public:
}
protected:
LibExprTest()
LibExprTest(ref<Store> store, auto && makeEvalSettings)
: LibStoreTest()
, evalSettings(makeEvalSettings(readOnlyMode))
, state({}, store, fetchSettings, evalSettings, nullptr)
{
evalSettings.nixPath = {};
}
LibExprTest()
: LibExprTest(openStore("dummy://"), [](bool & readOnlyMode) {
EvalSettings settings{readOnlyMode};
settings.nixPath = {};
return settings;
})
{
}
Value eval(std::string input, bool forceValue = true)

1
src/libexpr-test-support/meson.build
View file

@ -31,7 +31,6 @@ rapidcheck = dependency('rapidcheck')
deps_public += rapidcheck
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'tests/value/context.cc',

38
src/libexpr-tests/eval.cc
View file

@ -3,6 +3,7 @@
#include "nix/expr/eval.hh"
#include "nix/expr/tests/libexpr.hh"
#include "nix/util/memory-source-accessor.hh"
namespace nix {
@ -174,4 +175,41 @@ TEST_F(EvalStateTest, getBuiltin_fail)
ASSERT_THROW(state.getBuiltin("nonexistent"), EvalError);
}
class PureEvalTest : public LibExprTest
{
public:
PureEvalTest()
: LibExprTest(openStore("dummy://", {{"read-only", "false"}}), [](bool & readOnlyMode) {
EvalSettings settings{readOnlyMode};
settings.pureEval = true;
settings.restrictEval = true;
return settings;
})
{
}
};
TEST_F(PureEvalTest, pathExists)
{
ASSERT_THAT(eval("builtins.pathExists /."), IsFalse());
ASSERT_THAT(eval("builtins.pathExists /nix"), IsFalse());
ASSERT_THAT(eval("builtins.pathExists /nix/store"), IsFalse());
{
std::string contents = "Lorem ipsum";
StringSource s{contents};
auto path = state.store->addToStoreFromDump(
s, "source", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256);
auto printed = store->printStorePath(path);
ASSERT_THROW(eval(fmt("builtins.readFile %s", printed)), RestrictedPathError);
ASSERT_THAT(eval(fmt("builtins.pathExists %s", printed)), IsFalse());
ASSERT_THROW(eval("builtins.readDir /."), RestrictedPathError);
state.allowPath(path); // FIXME: This shouldn't behave this way.
ASSERT_THAT(eval("builtins.readDir /."), IsAttrsOfSize(0));
}
}
} // namespace nix

6
src/libexpr-tests/main.cc
View file

@ -1,15 +1,19 @@
#include <gtest/gtest.h>
#include "nix/store/tests/test-main.hh"
#include "nix/util/config-global.hh"
using namespace nix;
int main(int argc, char ** argv)
{
auto res = testMainForBuidingPre(argc, argv);
if (!res)
if (res)
return res;
// For pipe operator tests in trivial.cc
experimentalFeatureSettings.set("experimental-features", "pipe-operators");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

3
src/libexpr-tests/meson.build
View file

@ -45,7 +45,6 @@ config_priv_h = configure_file(
)
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'derived-path.cc',
@ -83,7 +82,7 @@ this_exe = executable(
test(
meson.project_name(),
this_exe,
env : asan_test_options_env + {
env : {
'_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data',
},
protocol : 'gtest',

49
src/libexpr-tests/nix_api_expr.cc
View file

@ -423,6 +423,55 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk)
ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("badReturnThunk"));
}
static void primop_with_nix_err_key(
void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret)
{
nix_set_err_msg(context, NIX_ERR_KEY, "Test error from primop");
}
TEST_F(nix_api_expr_test, nix_expr_primop_nix_err_key_conversion)
{
// Test that NIX_ERR_KEY from a custom primop gets converted to a generic EvalError
//
// RATIONALE: NIX_ERR_KEY must not be propagated from custom primops because it would
// create semantic confusion. NIX_ERR_KEY indicates missing keys/indices in C API functions
// (like nix_get_attr_byname, nix_get_list_byidx). If custom primops could return NIX_ERR_KEY,
// an evaluation error would be indistinguishable from an actual missing attribute.
//
// For example, if nix_get_attr_byname returned NIX_ERR_KEY when the attribute is present
// but the value evaluation fails, callers expecting NIX_ERR_KEY to mean "missing attribute"
// would incorrectly handle evaluation failures as missing attributes. In places where
// missing attributes are tolerated (like optional attributes), this would cause the
// program to continue after swallowing the error, leading to silent failures.
PrimOp * primop = nix_alloc_primop(
ctx, primop_with_nix_err_key, 1, "testErrorPrimop", nullptr, "a test primop that sets NIX_ERR_KEY", nullptr);
assert_ctx_ok();
nix_value * primopValue = nix_alloc_value(ctx, state);
assert_ctx_ok();
nix_init_primop(ctx, primopValue, primop);
assert_ctx_ok();
nix_value * arg = nix_alloc_value(ctx, state);
assert_ctx_ok();
nix_init_int(ctx, arg, 42);
assert_ctx_ok();
nix_value * result = nix_alloc_value(ctx, state);
assert_ctx_ok();
nix_value_call(ctx, state, primopValue, arg, result);
// Verify that NIX_ERR_KEY gets converted to NIX_ERR_NIX_ERROR (generic evaluation error)
ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("Error from custom function"));
ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("Test error from primop"));
ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("testErrorPrimop"));
// Clean up
nix_gc_decref(ctx, primopValue);
nix_gc_decref(ctx, arg);
nix_gc_decref(ctx, result);
}
TEST_F(nix_api_expr_test, nix_value_call_multi_no_args)
{
nix_value * n = nix_alloc_value(ctx, state);

327
src/libexpr-tests/nix_api_value.cc
View file

@ -162,6 +162,114 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list)
nix_gc_decref(ctx, intValue);
}
TEST_F(nix_api_expr_test, nix_get_list_byidx_large_indices)
{
// Create a small list to test extremely large out-of-bounds access
ListBuilder * builder = nix_make_list_builder(ctx, state, 2);
nix_value * intValue = nix_alloc_value(ctx, state);
nix_init_int(ctx, intValue, 42);
nix_list_builder_insert(ctx, builder, 0, intValue);
nix_list_builder_insert(ctx, builder, 1, intValue);
nix_make_list(ctx, builder, value);
nix_list_builder_free(builder);
// Test extremely large indices that would definitely crash without bounds checking
ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 1000000));
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, UINT_MAX / 2));
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, UINT_MAX / 2 + 1000000));
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
// Clean up
nix_gc_decref(ctx, intValue);
}
TEST_F(nix_api_expr_test, nix_get_list_byidx_lazy)
{
// Create a list with a throwing lazy element, an already-evaluated int, and a lazy function call
// 1. Throwing lazy element - create a function application thunk that will throw when forced
nix_value * throwingFn = nix_alloc_value(ctx, state);
nix_value * throwingValue = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(
ctx,
state,
R"(
_: throw "This should not be evaluated by the lazy accessor"
)",
"<test>",
throwingFn);
assert_ctx_ok();
nix_init_apply(ctx, throwingValue, throwingFn, throwingFn);
assert_ctx_ok();
// 2. Already evaluated int (not lazy)
nix_value * intValue = nix_alloc_value(ctx, state);
nix_init_int(ctx, intValue, 42);
assert_ctx_ok();
// 3. Lazy function application that would compute increment 5 = 6
nix_value * lazyApply = nix_alloc_value(ctx, state);
nix_value * incrementFn = nix_alloc_value(ctx, state);
nix_value * argFive = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(ctx, state, "x: x + 1", "<test>", incrementFn);
assert_ctx_ok();
nix_init_int(ctx, argFive, 5);
// Create a lazy application: (x: x + 1) 5
nix_init_apply(ctx, lazyApply, incrementFn, argFive);
assert_ctx_ok();
ListBuilder * builder = nix_make_list_builder(ctx, state, 3);
nix_list_builder_insert(ctx, builder, 0, throwingValue);
nix_list_builder_insert(ctx, builder, 1, intValue);
nix_list_builder_insert(ctx, builder, 2, lazyApply);
nix_make_list(ctx, builder, value);
nix_list_builder_free(builder);
// Test 1: Lazy accessor should return the throwing element without forcing evaluation
nix_value * lazyThrowingElement = nix_get_list_byidx_lazy(ctx, value, state, 0);
assert_ctx_ok();
ASSERT_NE(nullptr, lazyThrowingElement);
// Verify the element is still lazy by checking that forcing it throws
nix_value_force(ctx, state, lazyThrowingElement);
assert_ctx_err();
ASSERT_THAT(
nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor"));
// Test 2: Lazy accessor should return the already-evaluated int
nix_value * intElement = nix_get_list_byidx_lazy(ctx, value, state, 1);
assert_ctx_ok();
ASSERT_NE(nullptr, intElement);
ASSERT_EQ(42, nix_get_int(ctx, intElement));
// Test 3: Lazy accessor should return the lazy function application without forcing
nix_value * lazyFunctionElement = nix_get_list_byidx_lazy(ctx, value, state, 2);
assert_ctx_ok();
ASSERT_NE(nullptr, lazyFunctionElement);
// Force the lazy function application - should compute 5 + 1 = 6
nix_value_force(ctx, state, lazyFunctionElement);
assert_ctx_ok();
ASSERT_EQ(6, nix_get_int(ctx, lazyFunctionElement));
// Clean up
nix_gc_decref(ctx, throwingFn);
nix_gc_decref(ctx, throwingValue);
nix_gc_decref(ctx, intValue);
nix_gc_decref(ctx, lazyApply);
nix_gc_decref(ctx, incrementFn);
nix_gc_decref(ctx, argFive);
nix_gc_decref(ctx, lazyThrowingElement);
nix_gc_decref(ctx, intElement);
nix_gc_decref(ctx, lazyFunctionElement);
}
TEST_F(nix_api_expr_test, nix_build_and_init_attr_invalid)
{
ASSERT_EQ(nullptr, nix_get_attr_byname(ctx, nullptr, state, 0));
@ -244,6 +352,225 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr)
free(out_name);
}
TEST_F(nix_api_expr_test, nix_get_attr_byidx_large_indices)
{
// Create a small attribute set to test extremely large out-of-bounds access
const char ** out_name = (const char **) malloc(sizeof(char *));
BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 2);
nix_value * intValue = nix_alloc_value(ctx, state);
nix_init_int(ctx, intValue, 42);
nix_bindings_builder_insert(ctx, builder, "test", intValue);
nix_make_attrs(ctx, value, builder);
nix_bindings_builder_free(builder);
// Test extremely large indices that would definitely crash without bounds checking
ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, 1000000, out_name));
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, UINT_MAX / 2, out_name));
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, UINT_MAX / 2 + 1000000, out_name));
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
// Test nix_get_attr_name_byidx with large indices too
ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, 1000000));
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, UINT_MAX / 2));
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, UINT_MAX / 2 + 1000000));
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
// Clean up
nix_gc_decref(ctx, intValue);
free(out_name);
}
TEST_F(nix_api_expr_test, nix_get_attr_byname_lazy)
{
// Create an attribute set with a throwing lazy attribute, an already-evaluated int, and a lazy function call
// 1. Throwing lazy element - create a function application thunk that will throw when forced
nix_value * throwingFn = nix_alloc_value(ctx, state);
nix_value * throwingValue = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(
ctx,
state,
R"(
_: throw "This should not be evaluated by the lazy accessor"
)",
"<test>",
throwingFn);
assert_ctx_ok();
nix_init_apply(ctx, throwingValue, throwingFn, throwingFn);
assert_ctx_ok();
// 2. Already evaluated int (not lazy)
nix_value * intValue = nix_alloc_value(ctx, state);
nix_init_int(ctx, intValue, 42);
assert_ctx_ok();
// 3. Lazy function application that would compute increment 7 = 8
nix_value * lazyApply = nix_alloc_value(ctx, state);
nix_value * incrementFn = nix_alloc_value(ctx, state);
nix_value * argSeven = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(ctx, state, "x: x + 1", "<test>", incrementFn);
assert_ctx_ok();
nix_init_int(ctx, argSeven, 7);
// Create a lazy application: (x: x + 1) 7
nix_init_apply(ctx, lazyApply, incrementFn, argSeven);
assert_ctx_ok();
BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 3);
nix_bindings_builder_insert(ctx, builder, "throwing", throwingValue);
nix_bindings_builder_insert(ctx, builder, "normal", intValue);
nix_bindings_builder_insert(ctx, builder, "lazy", lazyApply);
nix_make_attrs(ctx, value, builder);
nix_bindings_builder_free(builder);
// Test 1: Lazy accessor should return the throwing attribute without forcing evaluation
nix_value * lazyThrowingAttr = nix_get_attr_byname_lazy(ctx, value, state, "throwing");
assert_ctx_ok();
ASSERT_NE(nullptr, lazyThrowingAttr);
// Verify the attribute is still lazy by checking that forcing it throws
nix_value_force(ctx, state, lazyThrowingAttr);
assert_ctx_err();
ASSERT_THAT(
nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor"));
// Test 2: Lazy accessor should return the already-evaluated int
nix_value * intAttr = nix_get_attr_byname_lazy(ctx, value, state, "normal");
assert_ctx_ok();
ASSERT_NE(nullptr, intAttr);
ASSERT_EQ(42, nix_get_int(ctx, intAttr));
// Test 3: Lazy accessor should return the lazy function application without forcing
nix_value * lazyFunctionAttr = nix_get_attr_byname_lazy(ctx, value, state, "lazy");
assert_ctx_ok();
ASSERT_NE(nullptr, lazyFunctionAttr);
// Force the lazy function application - should compute 7 + 1 = 8
nix_value_force(ctx, state, lazyFunctionAttr);
assert_ctx_ok();
ASSERT_EQ(8, nix_get_int(ctx, lazyFunctionAttr));
// Test 4: Missing attribute should return NULL with NIX_ERR_KEY
nix_value * missingAttr = nix_get_attr_byname_lazy(ctx, value, state, "nonexistent");
ASSERT_EQ(nullptr, missingAttr);
ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx));
// Clean up
nix_gc_decref(ctx, throwingFn);
nix_gc_decref(ctx, throwingValue);
nix_gc_decref(ctx, intValue);
nix_gc_decref(ctx, lazyApply);
nix_gc_decref(ctx, incrementFn);
nix_gc_decref(ctx, argSeven);
nix_gc_decref(ctx, lazyThrowingAttr);
nix_gc_decref(ctx, intAttr);
nix_gc_decref(ctx, lazyFunctionAttr);
}
TEST_F(nix_api_expr_test, nix_get_attr_byidx_lazy)
{
// Create an attribute set with a throwing lazy attribute, an already-evaluated int, and a lazy function call
// 1. Throwing lazy element - create a function application thunk that will throw when forced
nix_value * throwingFn = nix_alloc_value(ctx, state);
nix_value * throwingValue = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(
ctx,
state,
R"(
_: throw "This should not be evaluated by the lazy accessor"
)",
"<test>",
throwingFn);
assert_ctx_ok();
nix_init_apply(ctx, throwingValue, throwingFn, throwingFn);
assert_ctx_ok();
// 2. Already evaluated int (not lazy)
nix_value * intValue = nix_alloc_value(ctx, state);
nix_init_int(ctx, intValue, 99);
assert_ctx_ok();
// 3. Lazy function application that would compute increment 10 = 11
nix_value * lazyApply = nix_alloc_value(ctx, state);
nix_value * incrementFn = nix_alloc_value(ctx, state);
nix_value * argTen = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(ctx, state, "x: x + 1", "<test>", incrementFn);
assert_ctx_ok();
nix_init_int(ctx, argTen, 10);
// Create a lazy application: (x: x + 1) 10
nix_init_apply(ctx, lazyApply, incrementFn, argTen);
assert_ctx_ok();
BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 3);
nix_bindings_builder_insert(ctx, builder, "a_throwing", throwingValue);
nix_bindings_builder_insert(ctx, builder, "b_normal", intValue);
nix_bindings_builder_insert(ctx, builder, "c_lazy", lazyApply);
nix_make_attrs(ctx, value, builder);
nix_bindings_builder_free(builder);
// Proper usage: first get the size and gather all attributes into a map
unsigned int attrCount = nix_get_attrs_size(ctx, value);
assert_ctx_ok();
ASSERT_EQ(3u, attrCount);
// Gather all attributes into a map (proper contract usage)
std::map<std::string, nix_value *> attrMap;
const char * name;
for (unsigned int i = 0; i < attrCount; i++) {
nix_value * attr = nix_get_attr_byidx_lazy(ctx, value, state, i, &name);
assert_ctx_ok();
ASSERT_NE(nullptr, attr);
attrMap[std::string(name)] = attr;
}
// Now test the gathered attributes
ASSERT_EQ(3u, attrMap.size());
ASSERT_TRUE(attrMap.count("a_throwing"));
ASSERT_TRUE(attrMap.count("b_normal"));
ASSERT_TRUE(attrMap.count("c_lazy"));
// Test 1: Throwing attribute should be lazy
nix_value * throwingAttr = attrMap["a_throwing"];
nix_value_force(ctx, state, throwingAttr);
assert_ctx_err();
ASSERT_THAT(
nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor"));
// Test 2: Normal attribute should be already evaluated
nix_value * normalAttr = attrMap["b_normal"];
ASSERT_EQ(99, nix_get_int(ctx, normalAttr));
// Test 3: Lazy function should compute when forced
nix_value * lazyAttr = attrMap["c_lazy"];
nix_value_force(ctx, state, lazyAttr);
assert_ctx_ok();
ASSERT_EQ(11, nix_get_int(ctx, lazyAttr));
// Clean up
nix_gc_decref(ctx, throwingFn);
nix_gc_decref(ctx, throwingValue);
nix_gc_decref(ctx, intValue);
nix_gc_decref(ctx, lazyApply);
nix_gc_decref(ctx, incrementFn);
nix_gc_decref(ctx, argTen);
for (auto & pair : attrMap) {
nix_gc_decref(ctx, pair.second);
}
}
TEST_F(nix_api_expr_test, nix_value_init)
{
// Setup

1
src/libexpr-tests/package.nix
View file

@ -62,7 +62,6 @@ mkMesonExecutable (finalAttrs: {
mkdir -p "$HOME"
''
+ ''
export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0
export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
touch $out

6
src/libexpr-tests/primops.cc
View file

@ -642,7 +642,7 @@ class ToStringPrimOpTest : public PrimOpTest,
TEST_P(ToStringPrimOpTest, toString)
{
const auto [input, output] = GetParam();
const auto & [input, output] = GetParam();
auto v = eval(input);
ASSERT_THAT(v, IsStringEq(output));
}
@ -798,7 +798,7 @@ class CompareVersionsPrimOpTest : public PrimOpTest,
TEST_P(CompareVersionsPrimOpTest, compareVersions)
{
auto [expression, expectation] = GetParam();
const auto & [expression, expectation] = GetParam();
auto v = eval(expression);
ASSERT_THAT(v, IsIntEq(expectation));
}
@ -834,7 +834,7 @@ class ParseDrvNamePrimOpTest
TEST_P(ParseDrvNamePrimOpTest, parseDrvName)
{
auto [input, expectedName, expectedVersion] = GetParam();
const auto & [input, expectedName, expectedVersion] = GetParam();
const auto expr = fmt("builtins.parseDrvName \"%1%\"", input);
auto v = eval(expr);
ASSERT_THAT(v, IsAttrsOfSize(2));

10
src/libexpr/attr-set.cc
View file

@ -10,27 +10,27 @@ Bindings Bindings::emptyBindings;
/* Allocate a new array of attributes for an attribute set with a specific
capacity. The space is implicitly reserved after the Bindings
structure. */
Bindings * EvalState::allocBindings(size_t capacity)
Bindings * EvalMemory::allocBindings(size_t capacity)
{
if (capacity == 0)
return &Bindings::emptyBindings;
if (capacity > std::numeric_limits<Bindings::size_type>::max())
throw Error("attribute set of size %d is too big", capacity);
nrAttrsets++;
nrAttrsInAttrsets += capacity;
stats.nrAttrsets++;
stats.nrAttrsInAttrsets += capacity;
return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings();
}
Value & BindingsBuilder::alloc(Symbol name, PosIdx pos)
{
auto value = state.get().allocValue();
auto value = mem.get().allocValue();
bindings->push_back(Attr(name, value, pos));
return *value;
}
Value & BindingsBuilder::alloc(std::string_view name, PosIdx pos)
{
return alloc(state.get().symbols.create(name), pos);
return alloc(symbols.get().create(name), pos);
}
void Bindings::sort()

2
src/libexpr/eval-profiler.cc
View file

@ -324,7 +324,7 @@ void SampleStack::saveProfile()
std::visit([&](auto && info) { info.symbolize(state, os, posCache); }, pos);
}
os << " " << count;
writeLine(profileFd.get(), std::move(os).str());
writeLine(profileFd.get(), os.str());
/* Clear ostringstream. */
os.str("");
os.clear();

324
src/libexpr/eval.cc
View file

@ -17,6 +17,7 @@
#include "nix/expr/print.hh"
#include "nix/fetchers/filtering-source-accessor.hh"
#include "nix/util/memory-source-accessor.hh"
#include "nix/util/mounted-source-accessor.hh"
#include "nix/expr/gc-small-vector.hh"
#include "nix/util/url.hh"
#include "nix/fetchers/fetch-to-store.hh"
@ -38,6 +39,7 @@
#include <nlohmann/json.hpp>
#include <boost/container/small_vector.hpp>
#include <boost/unordered/concurrent_flat_map.hpp>
#include "nix/util/strings-inline.hh"
@ -192,6 +194,15 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
static constexpr size_t BASE_ENV_SIZE = 128;
EvalMemory::EvalMemory()
#if NIX_USE_BOEHMGC
: valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
, env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
#endif
{
assertGCInitialized();
}
EvalState::EvalState(
const LookupPath & lookupPathFromArguments,
ref<Store> store,
@ -224,22 +235,18 @@ EvalState::EvalState(
*/
{CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
}))
, rootFS(({
, rootFS([&] {
/* In pure eval mode, we provide a filesystem that only
contains the Nix store.
If we have a chroot store and pure eval is not enabled,
use a union accessor to make the chroot store available
at its logical location while still having the
Otherwise, use a union accessor to make the augmented store
available at its logical location while still having the
underlying directory available. This is necessary for
instance if we're evaluating a file from the physical
/nix/store while using a chroot store. */
auto accessor = getFSSourceAccessor();
auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
if (settings.pureEval || store->storeDir != realStoreDir) {
accessor = settings.pureEval ? storeFS : makeUnionSourceAccessor({accessor, storeFS});
}
/nix/store while using a chroot store, and also for lazy
mounted fetchTree. */
auto accessor = settings.pureEval ? storeFS.cast<SourceAccessor>()
: makeUnionSourceAccessor({getFSSourceAccessor(), storeFS});
/* Apply access control if needed. */
if (settings.restrictEval || settings.pureEval)
@ -250,11 +257,11 @@ EvalState::EvalState(
throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
});
accessor;
}))
return accessor;
}())
, corepkgsFS(make_ref<MemorySourceAccessor>())
, internalFS(make_ref<MemorySourceAccessor>())
, derivationInternal{corepkgsFS->addFile(
, derivationInternal{internalFS->addFile(
CanonPath("derivation-internal.nix"),
#include "primops/derivation.nix.gen.hh"
)}
@ -264,14 +271,15 @@ EvalState::EvalState(
, debugRepl(nullptr)
, debugStop(false)
, trylevel(0)
, srcToStore(make_ref<decltype(srcToStore)::element_type>())
, importResolutionCache(make_ref<decltype(importResolutionCache)::element_type>())
, fileEvalCache(make_ref<decltype(fileEvalCache)::element_type>())
, regexCache(makeRegexCache())
#if NIX_USE_BOEHMGC
, valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
, env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
, baseEnvP(std::allocate_shared<Env *>(traceable_allocator<Env *>(), &allocEnv(BASE_ENV_SIZE)))
, baseEnvP(std::allocate_shared<Env *>(traceable_allocator<Env *>(), &mem.allocEnv(BASE_ENV_SIZE)))
, baseEnv(**baseEnvP)
#else
, baseEnv(allocEnv(BASE_ENV_SIZE))
, baseEnv(mem.allocEnv(BASE_ENV_SIZE))
#endif
, staticBaseEnv{std::make_shared<StaticEnv>(nullptr, nullptr)}
{
@ -280,9 +288,8 @@ EvalState::EvalState(
countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
assertGCInitialized();
static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
static_assert(sizeof(Counter) == 64, "counters must be 64 bytes");
/* Construct the Nix expression search path. */
assert(lookupPath.elements.empty());
@ -329,7 +336,7 @@ EvalState::EvalState(
EvalState::~EvalState() {}
void EvalState::allowPath(const Path & path)
void EvalState::allowPathLegacy(const Path & path)
{
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
rootFS2->allowPrefix(CanonPath(path));
@ -577,7 +584,7 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
.name = name,
.arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
.args = {},
.doc = makeImmutableString(toView(s)), // NOTE: memory leak when compiled without GC
.doc = makeImmutableString(s.view()), // NOTE: memory leak when compiled without GC
};
}
if (isFunctor(v)) {
@ -876,11 +883,10 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
}
}
ListBuilder::ListBuilder(EvalState & state, size_t size)
ListBuilder::ListBuilder(size_t size)
: size(size)
, elems(size <= 2 ? inlineElems : (Value **) allocBytes(size * sizeof(Value *)))
{
state.nrListElems += size;
}
Value * EvalState::getBool(bool b)
@ -888,7 +894,7 @@ Value * EvalState::getBool(bool b)
return b ? &Value::vTrue : &Value::vFalse;
}
unsigned long nrThunks = 0;
static Counter nrThunks;
static inline void mkThunk(Value & v, Env & env, Expr * expr)
{
@ -979,10 +985,6 @@ void EvalState::mkSingleDerivedPathString(const SingleDerivedPath & p, Value & v
});
}
/* Create a thunk for the delayed computation of the given expression
in the given environment. But if the expression is a variable,
then look it up right away. This significantly reduces the number
of thunks allocated. */
Value * Expr::maybeThunk(EvalState & state, Env & env)
{
Value * v = state.allocValue();
@ -1026,63 +1028,90 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
return &v;
}
/**
* A helper `Expr` class to lets us parse and evaluate Nix expressions
* from a thunk, ensuring that every file is parsed/evaluated only
* once (via the thunk stored in `EvalState::fileEvalCache`).
*/
struct ExprParseFile : Expr, gc
{
// FIXME: make this a reference (see below).
SourcePath path;
bool mustBeTrivial;
ExprParseFile(SourcePath & path, bool mustBeTrivial)
: path(path)
, mustBeTrivial(mustBeTrivial)
{
}
void eval(EvalState & state, Env & env, Value & v) override
{
printTalkative("evaluating file '%s'", path);
auto e = state.parseExprFromFile(path);
try {
auto dts =
state.debugRepl
? makeDebugTraceStacker(
state, *e, state.baseEnv, e->getPos(), "while evaluating the file '%s':", path.to_string())
: nullptr;
// Enforce that 'flake.nix' is a direct attrset, not a
// computation.
if (mustBeTrivial && !(dynamic_cast<ExprAttrs *>(e)))
state.error<EvalError>("file '%s' must be an attribute set", path).debugThrow();
state.eval(e, v);
} catch (Error & e) {
state.addErrorTrace(e, "while evaluating the file '%s':", path.to_string());
throw;
}
}
};
void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
{
FileEvalCache::iterator i;
if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) {
v = i->second;
auto resolvedPath = getConcurrent(*importResolutionCache, path);
if (!resolvedPath) {
resolvedPath = resolveExprPath(path);
importResolutionCache->emplace(path, *resolvedPath);
}
if (auto v2 = getConcurrent(*fileEvalCache, *resolvedPath)) {
forceValue(**v2, noPos);
v = **v2;
return;
}
auto resolvedPath = resolveExprPath(path);
if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) {
v = i->second;
return;
}
Value * vExpr;
// FIXME: put ExprParseFile on the stack instead of the heap once
// https://github.com/NixOS/nix/pull/13930 is merged. That will ensure
// the post-condition that `expr` is unreachable after
// `forceValue()` returns.
auto expr = new ExprParseFile{*resolvedPath, mustBeTrivial};
printTalkative("evaluating file '%1%'", resolvedPath);
Expr * e = nullptr;
fileEvalCache->try_emplace_and_cvisit(
*resolvedPath,
nullptr,
[&](auto & i) {
vExpr = allocValue();
vExpr->mkThunk(&baseEnv, expr);
i.second = vExpr;
},
[&](auto & i) { vExpr = i.second; });
auto j = fileParseCache.find(resolvedPath);
if (j != fileParseCache.end())
e = j->second;
forceValue(*vExpr, noPos);
if (!e)
e = parseExprFromFile(resolvedPath);
fileParseCache.emplace(resolvedPath, e);
try {
auto dts = debugRepl ? makeDebugTraceStacker(
*this,
*e,
this->baseEnv,
e->getPos(),
"while evaluating the file '%1%':",
resolvedPath.to_string())
: nullptr;
// Enforce that 'flake.nix' is a direct attrset, not a
// computation.
if (mustBeTrivial && !(dynamic_cast<ExprAttrs *>(e)))
error<EvalError>("file '%s' must be an attribute set", path).debugThrow();
eval(e, v);
} catch (Error & e) {
addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string());
throw;
}
fileEvalCache.emplace(resolvedPath, v);
if (path != resolvedPath)
fileEvalCache.emplace(path, v);
v = *vExpr;
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileEvalCache.rehash(0);
fileParseCache.clear();
fileParseCache.rehash(0);
importResolutionCache->clear();
fileEvalCache->clear();
inputCache->clear();
}
@ -1151,7 +1180,7 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v)
Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
{
Env & inheritEnv = state.allocEnv(inheritFromExprs->size());
Env & inheritEnv = state.mem.allocEnv(inheritFromExprs->size());
inheritEnv.up = &up;
Displacement displ = 0;
@ -1170,7 +1199,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
if (recursive) {
/* Create a new environment that contains the attributes in
this `rec'. */
Env & env2(state.allocEnv(attrs.size()));
Env & env2(state.mem.allocEnv(attrs.size()));
env2.up = &env;
dynamicEnv = &env2;
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
@ -1262,7 +1291,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
{
/* Create a new environment that contains the attributes in this
`let'. */
Env & env2(state.allocEnv(attrs->attrs.size()));
Env & env2(state.mem.allocEnv(attrs->attrs.size()));
env2.up = &env;
Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
@ -1305,7 +1334,7 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
v = *v2;
}
static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath)
static std::string showAttrPath(EvalState & state, Env & env, std::span<const AttrName> attrPath)
{
std::ostringstream out;
bool first = true;
@ -1341,10 +1370,10 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
env,
getPos(),
"while evaluating the attribute '%1%'",
showAttrPath(state, env, attrPath))
showAttrPath(state, env, getAttrPath()))
: nullptr;
for (auto & i : attrPath) {
for (auto & i : getAttrPath()) {
state.nrLookups++;
const Attr * j;
auto name = getName(i, state, env);
@ -1382,7 +1411,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
auto origin = std::get_if<SourcePath>(&pos2r.origin);
if (!(origin && *origin == state.derivationInternal))
state.addErrorTrace(
e, pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath));
e, pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, getAttrPath()));
}
throw;
}
@ -1393,13 +1422,13 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs)
{
Value vTmp;
Symbol name = getName(attrPath[attrPath.size() - 1], state, env);
Symbol name = getName(attrPathStart[nAttrPath - 1], state, env);
if (attrPath.size() == 1) {
if (nAttrPath == 1) {
e->eval(state, env, vTmp);
} else {
ExprSelect init(*this);
init.attrPath.pop_back();
init.nAttrPath--;
init.eval(state, env, vTmp);
}
attrs = vTmp;
@ -1468,7 +1497,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
ExprLambda & lambda(*vCur.lambda().fun);
auto size = (!lambda.arg ? 0 : 1) + (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
Env & env2(allocEnv(size));
Env & env2(mem.allocEnv(size));
env2.up = vCur.lambda().env;
Displacement displ = 0;
@ -1757,7 +1786,7 @@ https://nix.dev/manual/nix/stable/language/syntax.html#functions.)",
void ExprWith::eval(EvalState & state, Env & env, Value & v)
{
Env & env2(state.allocEnv(1));
Env & env2(state.mem.allocEnv(1));
env2.up = &env;
env2.values[0] = attrs->maybeThunk(state, env);
@ -1775,7 +1804,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
std::ostringstream out;
cond->show(state.symbols, out);
auto exprStr = toView(out);
auto exprStr = out.view();
if (auto eq = dynamic_cast<ExprOpEq *>(cond)) {
try {
@ -1839,12 +1868,8 @@ void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
|| state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator"));
}
void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
void ExprOpUpdate::eval(EvalState & state, Value & v, Value & v1, Value & v2)
{
Value v1, v2;
state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator");
state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator");
state.nrOpUpdates++;
const Bindings & bindings1 = *v1.attrs();
@ -1918,6 +1943,38 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
state.nrOpUpdateValuesCopied += v.attrs()->size();
}
void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
{
UpdateQueue q;
evalForUpdate(state, env, q);
v.mkAttrs(&Bindings::emptyBindings);
for (auto & rhs : std::views::reverse(q)) {
/* Remember that queue is sorted rightmost attrset first. */
eval(state, /*v=*/v, /*v1=*/v, /*v2=*/rhs);
}
}
void Expr::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx)
{
Value v;
state.evalAttrs(env, this, v, getPos(), errorCtx);
q.push_back(v);
}
void ExprOpUpdate::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q)
{
/* Output rightmost attrset first to the merge queue as the one
with the most priority. */
e2->evalForUpdate(state, env, q, "in the right operand of the update (//) operator");
e1->evalForUpdate(state, env, q, "in the left operand of the update (//) operator");
}
void ExprOpUpdate::evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx)
{
evalForUpdate(state, env, q);
}
void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
{
Value v1;
@ -2401,9 +2458,10 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
if (nix::isDerivation(path.path.abs()))
error<EvalError>("file names are not allowed to end in '%1%'", drvExtension).debugThrow();
std::optional<StorePath> dstPath;
if (!srcToStore.cvisit(path, [&dstPath](const auto & kv) { dstPath.emplace(kv.second); })) {
dstPath.emplace(fetchToStore(
auto dstPathCached = getConcurrent(*srcToStore, path);
auto dstPath = dstPathCached ? *dstPathCached : [&]() {
auto dstPath = fetchToStore(
fetchSettings,
*store,
path.resolveSymlinks(SymlinkResolution::Ancestors),
@ -2411,14 +2469,15 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
path.baseName(),
ContentAddressMethod::Raw::NixArchive,
nullptr,
repair));
allowPath(*dstPath);
srcToStore.try_emplace(path, *dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(*dstPath));
}
repair);
allowPath(dstPath);
srcToStore->try_emplace(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
return dstPath;
}();
context.insert(NixStringContextElem::Opaque{.path = *dstPath});
return *dstPath;
context.insert(NixStringContextElem::Opaque{.path = dstPath});
return dstPath;
}
SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
@ -2834,11 +2893,11 @@ bool EvalState::fullGC()
#endif
}
bool Counter::enabled = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
void EvalState::maybePrintStats()
{
bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
if (showStats) {
if (Counter::enabled) {
// Make the final heap size more deterministic.
#if NIX_USE_BOEHMGC
if (!fullGC()) {
@ -2854,10 +2913,12 @@ void EvalState::printStatistics()
std::chrono::microseconds cpuTimeDuration = getCpuUserTime();
float cpuTime = std::chrono::duration_cast<std::chrono::duration<float>>(cpuTimeDuration).count();
uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *);
uint64_t bLists = nrListElems * sizeof(Value *);
uint64_t bValues = nrValues * sizeof(Value);
uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
auto & memstats = mem.getStats();
uint64_t bEnvs = memstats.nrEnvs * sizeof(Env) + memstats.nrValuesInEnvs * sizeof(Value *);
uint64_t bLists = memstats.nrListElems * sizeof(Value *);
uint64_t bValues = memstats.nrValues * sizeof(Value);
uint64_t bAttrsets = memstats.nrAttrsets * sizeof(Bindings) + memstats.nrAttrsInAttrsets * sizeof(Attr);
#if NIX_USE_BOEHMGC
GC_word heapSize, totalBytes;
@ -2883,18 +2944,18 @@ void EvalState::printStatistics()
#endif
};
topObj["envs"] = {
{"number", nrEnvs},
{"elements", nrValuesInEnvs},
{"number", memstats.nrEnvs.load()},
{"elements", memstats.nrValuesInEnvs.load()},
{"bytes", bEnvs},
};
topObj["nrExprs"] = Expr::nrExprs;
topObj["nrExprs"] = Expr::nrExprs.load();
topObj["list"] = {
{"elements", nrListElems},
{"elements", memstats.nrListElems.load()},
{"bytes", bLists},
{"concats", nrListConcats},
{"concats", nrListConcats.load()},
};
topObj["values"] = {
{"number", nrValues},
{"number", memstats.nrValues.load()},
{"bytes", bValues},
};
topObj["symbols"] = {
@ -2902,9 +2963,9 @@ void EvalState::printStatistics()
{"bytes", symbols.totalSize()},
};
topObj["sets"] = {
{"number", nrAttrsets},
{"number", memstats.nrAttrsets.load()},
{"bytes", bAttrsets},
{"elements", nrAttrsInAttrsets},
{"elements", memstats.nrAttrsInAttrsets.load()},
};
topObj["sizes"] = {
{"Env", sizeof(Env)},
@ -2912,13 +2973,13 @@ void EvalState::printStatistics()
{"Bindings", sizeof(Bindings)},
{"Attr", sizeof(Attr)},
};
topObj["nrOpUpdates"] = nrOpUpdates;
topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied;
topObj["nrThunks"] = nrThunks;
topObj["nrAvoided"] = nrAvoided;
topObj["nrLookups"] = nrLookups;
topObj["nrPrimOpCalls"] = nrPrimOpCalls;
topObj["nrFunctionCalls"] = nrFunctionCalls;
topObj["nrOpUpdates"] = nrOpUpdates.load();
topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied.load();
topObj["nrThunks"] = nrThunks.load();
topObj["nrAvoided"] = nrAvoided.load();
topObj["nrLookups"] = nrLookups.load();
topObj["nrPrimOpCalls"] = nrPrimOpCalls.load();
topObj["nrFunctionCalls"] = nrFunctionCalls.load();
#if NIX_USE_BOEHMGC
topObj["gc"] = {
{"heapSize", heapSize},
@ -3065,6 +3126,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
auto res = (r / CanonPath(suffix)).resolveSymlinks();
if (res.pathExists())
return res;
// Backward compatibility hack: throw an exception if access
// to this path is not allowed.
if (auto accessor = res.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
accessor->checkAccess(res.path);
}
if (hasPrefix(path, "nix/"))
@ -3119,7 +3185,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
/* Allow access to paths in the search path. */
if (initAccessControl) {
allowPath(path.path.abs());
allowPathLegacy(path.path.abs());
if (store->isInStore(path.path.abs())) {
try {
allowClosure(store->toStorePath(path.path.abs()).first);
@ -3131,6 +3197,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
if (path.resolveSymlinks().pathExists())
return finish(std::move(path));
else {
// Backward compatibility hack: throw an exception if access
// to this path is not allowed.
if (auto accessor = path.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
accessor->checkAccess(path.path);
logWarning({.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)});
}
}
@ -3149,7 +3220,8 @@ Expr * EvalState::parse(
docComments = &it->second;
}
auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS);
auto result = parseExprFromBuf(
text, length, origin, basePath, mem.exprs.alloc, symbols, settings, positions, *docComments, rootFS);
result->bindVars(*this, staticEnv);

14
src/libexpr/include/nix/expr/attr-set.hh
View file

@ -13,7 +13,7 @@
namespace nix {
class EvalState;
class EvalMemory;
struct Value;
/**
@ -426,7 +426,7 @@ public:
return res;
}
friend class EvalState;
friend class EvalMemory;
};
static_assert(std::forward_iterator<Bindings::iterator>);
@ -448,12 +448,13 @@ private:
Bindings * bindings;
Bindings::size_type capacity_;
friend class EvalState;
friend class EvalMemory;
BindingsBuilder(EvalState & state, Bindings * bindings, size_type capacity)
BindingsBuilder(EvalMemory & mem, SymbolTable & symbols, Bindings * bindings, size_type capacity)
: bindings(bindings)
, capacity_(capacity)
, state(state)
, mem(mem)
, symbols(symbols)
{
}
@ -471,7 +472,8 @@ private:
}
public:
std::reference_wrapper<EvalState> state;
std::reference_wrapper<EvalMemory> mem;
std::reference_wrapper<SymbolTable> symbols;
void insert(Symbol name, Value * value, PosIdx pos = noPos)
{

70
src/libexpr/include/nix/expr/counter.hh Normal file
View file

@ -0,0 +1,70 @@
#pragma once
#include <atomic>
#include <cstdint>
namespace nix {
/**
* An atomic counter aligned on a cache line to prevent false sharing.
* The counter is only enabled when the `NIX_SHOW_STATS` environment
* variable is set. This is to prevent contention on these counters
* when multi-threaded evaluation is enabled.
*/
struct alignas(64) Counter
{
using value_type = uint64_t;
std::atomic<value_type> inner{0};
static bool enabled;
Counter() {}
operator value_type() const noexcept
{
return inner;
}
void operator=(value_type n) noexcept
{
inner = n;
}
value_type load() const noexcept
{
return inner;
}
value_type operator++() noexcept
{
return enabled ? ++inner : 0;
}
value_type operator++(int) noexcept
{
return enabled ? inner++ : 0;
}
value_type operator--() noexcept
{
return enabled ? --inner : 0;
}
value_type operator--(int) noexcept
{
return enabled ? inner-- : 0;
}
value_type operator+=(value_type n) noexcept
{
return enabled ? inner += n : 0;
}
value_type operator-=(value_type n) noexcept
{
return enabled ? inner -= n : 0;
}
};
} // namespace nix

10
src/libexpr/include/nix/expr/eval-inline.hh
View file

@ -26,7 +26,7 @@ inline void * allocBytes(size_t n)
}
[[gnu::always_inline]]
Value * EvalState::allocValue()
Value * EvalMemory::allocValue()
{
#if NIX_USE_BOEHMGC
/* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
@ -48,15 +48,15 @@ Value * EvalState::allocValue()
void * p = allocBytes(sizeof(Value));
#endif
nrValues++;
stats.nrValues++;
return (Value *) p;
}
[[gnu::always_inline]]
Env & EvalState::allocEnv(size_t size)
Env & EvalMemory::allocEnv(size_t size)
{
nrEnvs++;
nrValuesInEnvs += size;
stats.nrEnvs++;
stats.nrValuesInEnvs += size;
Env * env;

171
src/libexpr/include/nix/expr/eval.hh
View file

@ -16,12 +16,14 @@
#include "nix/expr/search-path.hh"
#include "nix/expr/repl-exit-status.hh"
#include "nix/util/ref.hh"
#include "nix/expr/counter.hh"
// For `NIX_USE_BOEHMGC`, and if that's set, `GC_THREADS`
#include "nix/expr/config.hh"
#include <boost/unordered/concurrent_flat_map.hpp>
#include <boost/unordered/unordered_flat_map.hpp>
#include <boost/unordered/concurrent_flat_map_fwd.hpp>
#include <map>
#include <optional>
#include <functional>
@ -40,6 +42,7 @@ class Store;
namespace fetchers {
struct Settings;
struct InputCache;
struct Input;
} // namespace fetchers
struct EvalSettings;
class EvalState;
@ -47,6 +50,7 @@ class StorePath;
struct SingleDerivedPath;
enum RepairFlag : bool;
struct MemorySourceAccessor;
struct MountedSourceAccessor;
namespace eval_cache {
class EvalCache;
@ -299,6 +303,68 @@ struct StaticEvalSymbols
}
};
class EvalMemory
{
#if NIX_USE_BOEHMGC
/**
* Allocation cache for GC'd Value objects.
*/
std::shared_ptr<void *> valueAllocCache;
/**
* Allocation cache for size-1 Env objects.
*/
std::shared_ptr<void *> env1AllocCache;
#endif
public:
struct Statistics
{
Counter nrEnvs;
Counter nrValuesInEnvs;
Counter nrValues;
Counter nrAttrsets;
Counter nrAttrsInAttrsets;
Counter nrListElems;
};
EvalMemory();
EvalMemory(const EvalMemory &) = delete;
EvalMemory(EvalMemory &&) = delete;
EvalMemory & operator=(const EvalMemory &) = delete;
EvalMemory & operator=(EvalMemory &&) = delete;
inline Value * allocValue();
inline Env & allocEnv(size_t size);
Bindings * allocBindings(size_t capacity);
BindingsBuilder buildBindings(SymbolTable & symbols, size_t capacity)
{
return BindingsBuilder(*this, symbols, allocBindings(capacity), capacity);
}
ListBuilder buildList(size_t size)
{
stats.nrListElems += size;
return ListBuilder(size);
}
const Statistics & getStats() const &
{
return stats;
}
/**
* Storage for the AST nodes
*/
Exprs exprs;
private:
Statistics stats;
};
class EvalState : public std::enable_shared_from_this<EvalState>
{
public:
@ -309,6 +375,8 @@ public:
SymbolTable symbols;
PosTable positions;
EvalMemory mem;
/**
* If set, force copying files to the Nix store even if they
* already exist there.
@ -318,7 +386,7 @@ public:
/**
* The accessor corresponding to `store`.
*/
const ref<SourceAccessor> storeFS;
const ref<MountedSourceAccessor> storeFS;
/**
* The accessor for the root filesystem.
@ -403,37 +471,30 @@ private:
/* Cache for calls to addToStore(); maps source paths to the store
paths. */
boost::concurrent_flat_map<SourcePath, StorePath, std::hash<SourcePath>> srcToStore;
ref<boost::concurrent_flat_map<SourcePath, StorePath>> srcToStore;
/**
* A cache from path names to parse trees.
* A cache that maps paths to "resolved" paths for importing Nix
* expressions, i.e. `/foo` to `/foo/default.nix`.
*/
typedef boost::unordered_flat_map<
SourcePath,
Expr *,
std::hash<SourcePath>,
std::equal_to<SourcePath>,
traceable_allocator<std::pair<const SourcePath, Expr *>>>
FileParseCache;
FileParseCache fileParseCache;
ref<boost::concurrent_flat_map<SourcePath, SourcePath>> importResolutionCache;
/**
* A cache from path names to values.
* A cache from resolved paths to values.
*/
typedef boost::unordered_flat_map<
ref<boost::concurrent_flat_map<
SourcePath,
Value,
Value *,
std::hash<SourcePath>,
std::equal_to<SourcePath>,
traceable_allocator<std::pair<const SourcePath, Value>>>
FileEvalCache;
FileEvalCache fileEvalCache;
traceable_allocator<std::pair<const SourcePath, Value *>>>>
fileEvalCache;
/**
* Associate source positions of certain AST nodes with their preceding doc comment, if they have one.
* Grouped by file.
*/
boost::unordered_flat_map<SourcePath, DocCommentMap, std::hash<SourcePath>> positionToDocComment;
boost::unordered_flat_map<SourcePath, DocCommentMap> positionToDocComment;
LookupPath lookupPath;
@ -445,28 +506,32 @@ private:
*/
std::shared_ptr<RegexCache> regexCache;
#if NIX_USE_BOEHMGC
/**
* Allocation cache for GC'd Value objects.
*/
std::shared_ptr<void *> valueAllocCache;
/**
* Allocation cache for size-1 Env objects.
*/
std::shared_ptr<void *> env1AllocCache;
#endif
public:
/**
* @param lookupPath Only used during construction.
* @param store The store to use for instantiation
* @param fetchSettings Must outlive the lifetime of this EvalState!
* @param settings Must outlive the lifetime of this EvalState!
* @param buildStore The store to use for builds ("import from derivation", C API `nix_string_realise`)
*/
EvalState(
const LookupPath & _lookupPath,
const LookupPath & lookupPath,
ref<Store> store,
const fetchers::Settings & fetchSettings,
const EvalSettings & settings,
std::shared_ptr<Store> buildStore = nullptr);
~EvalState();
/**
* A wrapper around EvalMemory::allocValue() to avoid code churn when it
* was introduced.
*/
inline Value * allocValue()
{
return mem.allocValue();
}
LookupPath getLookupPath()
{
return lookupPath;
@ -494,8 +559,11 @@ public:
/**
* Allow access to a path.
*
* Only for restrict eval: pure eval just whitelist store paths,
* never arbitrary paths.
*/
void allowPath(const Path & path);
void allowPathLegacy(const Path & path);
/**
* Allow access to a store path. Note that this gets remapped to
@ -515,6 +583,11 @@ public:
void checkURI(const std::string & uri);
/**
* Mount an input on the Nix store.
*/
StorePath mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor);
/**
* Parse a Nix expression from the specified file.
*/
@ -835,22 +908,14 @@ public:
*/
void autoCallFunction(const Bindings & args, Value & fun, Value & res);
/**
* Allocation primitives.
*/
inline Value * allocValue();
inline Env & allocEnv(size_t size);
Bindings * allocBindings(size_t capacity);
BindingsBuilder buildBindings(size_t capacity)
{
return BindingsBuilder(*this, allocBindings(capacity), capacity);
return mem.buildBindings(symbols, capacity);
}
ListBuilder buildList(size_t size)
{
return ListBuilder(*this, size);
return mem.buildList(size);
}
/**
@ -967,19 +1032,13 @@ private:
*/
std::string mkSingleDerivedPathStringRaw(const SingleDerivedPath & p);
unsigned long nrEnvs = 0;
unsigned long nrValuesInEnvs = 0;
unsigned long nrValues = 0;
unsigned long nrListElems = 0;
unsigned long nrLookups = 0;
unsigned long nrAttrsets = 0;
unsigned long nrAttrsInAttrsets = 0;
unsigned long nrAvoided = 0;
unsigned long nrOpUpdates = 0;
unsigned long nrOpUpdateValuesCopied = 0;
unsigned long nrListConcats = 0;
unsigned long nrPrimOpCalls = 0;
unsigned long nrFunctionCalls = 0;
Counter nrLookups;
Counter nrAvoided;
Counter nrOpUpdates;
Counter nrOpUpdateValuesCopied;
Counter nrListConcats;
Counter nrPrimOpCalls;
Counter nrFunctionCalls;
bool countCalls;

16
src/libexpr/include/nix/expr/gc-small-vector.hh
View file

@ -26,4 +26,20 @@ using SmallValueVector = SmallVector<Value *, nItems>;
template<size_t nItems>
using SmallTemporaryValueVector = SmallVector<Value, nItems>;
/**
* For functions where we do not expect deep recursion, we can use a sizable
* part of the stack a free allocation space.
*
* Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
*/
constexpr size_t nonRecursiveStackReservation = 128;
/**
* Functions that maybe applied to self-similar inputs, such as concatMap on a
* tree, should reserve a smaller part of the stack for allocation.
*
* Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
*/
constexpr size_t conservativeStackReservation = 16;
} // namespace nix

1
src/libexpr/include/nix/expr/meson.build
View file

@ -10,6 +10,7 @@ config_pub_h = configure_file(
headers = [ config_pub_h ] + files(
'attr-path.hh',
'attr-set.hh',
'counter.hh',
'eval-cache.hh',
'eval-error.hh',
'eval-gc.hh',

184
src/libexpr/include/nix/expr/nixexpr.hh
View file

@ -2,12 +2,17 @@
///@file
#include <map>
#include <span>
#include <vector>
#include <memory_resource>
#include <algorithm>
#include "nix/expr/gc-small-vector.hh"
#include "nix/expr/value.hh"
#include "nix/expr/symbol-table.hh"
#include "nix/expr/eval-error.hh"
#include "nix/util/pos-idx.hh"
#include "nix/expr/counter.hh"
namespace nix {
@ -76,9 +81,20 @@ struct AttrName
: expr(e) {};
};
static_assert(std::is_trivially_copy_constructible_v<AttrName>);
typedef std::vector<AttrName> AttrPath;
std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath);
std::string showAttrPath(const SymbolTable & symbols, std::span<const AttrName> attrPath);
using UpdateQueue = SmallTemporaryValueVector<conservativeStackReservation>;
class Exprs
{
std::pmr::monotonic_buffer_resource buffer;
public:
std::pmr::polymorphic_allocator<char> alloc{&buffer};
};
/* Abstract syntax of Nix expressions. */
@ -89,7 +105,7 @@ struct Expr
Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
};
static unsigned long nrExprs;
static Counter nrExprs;
Expr()
{
@ -99,8 +115,25 @@ struct Expr
virtual ~Expr() {};
virtual void show(const SymbolTable & symbols, std::ostream & str) const;
virtual void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
/** Normal evaluation, implemented directly by all subclasses. */
virtual void eval(EvalState & state, Env & env, Value & v);
/**
* Create a thunk for the delayed computation of the given expression
* in the given environment. But if the expression is a variable,
* then look it up right away. This significantly reduces the number
* of thunks allocated.
*/
virtual Value * maybeThunk(EvalState & state, Env & env);
/**
* Only called when performing an attrset update: `//` or similar.
* Instead of writing to a Value &, this function writes to an UpdateQueue.
* This allows the expression to perform multiple updates in a delayed manner, gathering up all the updates before
* applying them.
*/
virtual void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx);
virtual void setName(Symbol name);
virtual void setDocComment(DocComment docComment) {};
@ -152,13 +185,28 @@ struct ExprFloat : Expr
struct ExprString : Expr
{
std::string s;
Value v;
ExprString(std::string && s)
: s(std::move(s))
/**
* This is only for strings already allocated in our polymorphic allocator,
* or that live at least that long (e.g. c++ string literals)
*/
ExprString(const char * s)
{
v.mkStringNoCopy(this->s.data());
v.mkStringNoCopy(s);
};
ExprString(std::pmr::polymorphic_allocator<char> & alloc, std::string_view sv)
{
auto len = sv.length();
if (len == 0) {
v.mkStringNoCopy("");
return;
}
char * s = alloc.allocate(len + 1);
sv.copy(s, len);
s[len] = '\0';
v.mkStringNoCopy(s);
};
Value * maybeThunk(EvalState & state, Env & env) override;
@ -168,14 +216,16 @@ struct ExprString : Expr
struct ExprPath : Expr
{
ref<SourceAccessor> accessor;
std::string s;
Value v;
ExprPath(ref<SourceAccessor> accessor, std::string s)
ExprPath(std::pmr::polymorphic_allocator<char> & alloc, ref<SourceAccessor> accessor, std::string_view sv)
: accessor(accessor)
, s(std::move(s))
{
v.mkPath(&*accessor, this->s.c_str());
auto len = sv.length();
char * s = alloc.allocate(len + 1);
sv.copy(s, len);
s[len] = '\0';
v.mkPath(&*accessor, s);
}
Value * maybeThunk(EvalState & state, Env & env) override;
@ -242,20 +292,33 @@ struct ExprInheritFrom : ExprVar
struct ExprSelect : Expr
{
PosIdx pos;
uint32_t nAttrPath;
Expr *e, *def;
AttrPath attrPath;
ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def)
AttrName * attrPathStart;
ExprSelect(
std::pmr::polymorphic_allocator<char> & alloc,
const PosIdx & pos,
Expr * e,
std::span<const AttrName> attrPath,
Expr * def)
: pos(pos)
, nAttrPath(attrPath.size())
, e(e)
, def(def)
, attrPath(std::move(attrPath)) {};
, attrPathStart(alloc.allocate_object<AttrName>(nAttrPath))
{
std::ranges::copy(attrPath, attrPathStart);
};
ExprSelect(const PosIdx & pos, Expr * e, Symbol name)
ExprSelect(std::pmr::polymorphic_allocator<char> & alloc, const PosIdx & pos, Expr * e, Symbol name)
: pos(pos)
, nAttrPath(1)
, e(e)
, def(0)
, attrPathStart((alloc.allocate_object<AttrName>()))
{
attrPath.push_back(AttrName(name));
*attrPathStart = AttrName(name);
};
PosIdx getPos() const override
@ -263,6 +326,11 @@ struct ExprSelect : Expr
return pos;
}
std::span<const AttrName> getAttrPath() const
{
return {attrPathStart, nAttrPath};
}
/**
* Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl.
*
@ -280,10 +348,14 @@ struct ExprSelect : Expr
struct ExprOpHasAttr : Expr
{
Expr * e;
AttrPath attrPath;
ExprOpHasAttr(Expr * e, AttrPath attrPath)
std::span<AttrName> attrPath;
ExprOpHasAttr(std::pmr::polymorphic_allocator<char> & alloc, Expr * e, std::vector<AttrName> attrPath)
: e(e)
, attrPath(std::move(attrPath)) {};
, attrPath({alloc.allocate_object<AttrName>(attrPath.size()), attrPath.size()})
{
std::ranges::copy(attrPath, this->attrPath.begin());
};
PosIdx getPos() const override
{
@ -565,36 +637,39 @@ struct ExprOpNot : Expr
COMMON_METHODS
};
#define MakeBinOp(name, s) \
struct name : Expr \
{ \
PosIdx pos; \
Expr *e1, *e2; \
name(Expr * e1, Expr * e2) \
: e1(e1) \
, e2(e2) {}; \
name(const PosIdx & pos, Expr * e1, Expr * e2) \
: pos(pos) \
, e1(e1) \
, e2(e2) {}; \
void show(const SymbolTable & symbols, std::ostream & str) const override \
{ \
str << "("; \
e1->show(symbols, str); \
str << " " s " "; \
e2->show(symbols, str); \
str << ")"; \
} \
void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override \
{ \
e1->bindVars(es, env); \
e2->bindVars(es, env); \
} \
void eval(EvalState & state, Env & env, Value & v) override; \
PosIdx getPos() const override \
{ \
return pos; \
} \
#define MakeBinOpMembers(name, s) \
PosIdx pos; \
Expr *e1, *e2; \
name(Expr * e1, Expr * e2) \
: e1(e1) \
, e2(e2){}; \
name(const PosIdx & pos, Expr * e1, Expr * e2) \
: pos(pos) \
, e1(e1) \
, e2(e2){}; \
void show(const SymbolTable & symbols, std::ostream & str) const override \
{ \
str << "("; \
e1->show(symbols, str); \
str << " " s " "; \
e2->show(symbols, str); \
str << ")"; \
} \
void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override \
{ \
e1->bindVars(es, env); \
e2->bindVars(es, env); \
} \
void eval(EvalState & state, Env & env, Value & v) override; \
PosIdx getPos() const override \
{ \
return pos; \
}
#define MakeBinOp(name, s) \
struct name : Expr \
{ \
MakeBinOpMembers(name, s) \
}
MakeBinOp(ExprOpEq, "==");
@ -602,9 +677,20 @@ MakeBinOp(ExprOpNEq, "!=");
MakeBinOp(ExprOpAnd, "&&");
MakeBinOp(ExprOpOr, "||");
MakeBinOp(ExprOpImpl, "->");
MakeBinOp(ExprOpUpdate, "//");
MakeBinOp(ExprOpConcatLists, "++");
struct ExprOpUpdate : Expr
{
private:
/** Special case for merging of two attrsets. */
void eval(EvalState & state, Value & v, Value & v1, Value & v2);
void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q);
public:
MakeBinOpMembers(ExprOpUpdate, "//");
virtual void evalForUpdate(EvalState & state, Env & env, UpdateQueue & q, std::string_view errorCtx) override;
};
struct ExprConcatStrings : Expr
{
PosIdx pos;

7
src/libexpr/include/nix/expr/parser-state.hh
View file

@ -24,7 +24,6 @@ struct StringToken
}
};
// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y.
struct ParserLocation
{
int beginOffset;
@ -44,9 +43,6 @@ struct ParserLocation
beginOffset = stashedBeginOffset;
endOffset = stashedEndOffset;
}
/** Latest doc comment position, or 0. */
int doc_comment_first_column, doc_comment_last_column;
};
struct LexerState
@ -82,6 +78,7 @@ struct LexerState
struct ParserState
{
const LexerState & lexerState;
std::pmr::polymorphic_allocator<char> & alloc;
SymbolTable & symbols;
PosTable & positions;
Expr * result;
@ -327,7 +324,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
// Ignore empty strings for a minor optimisation and AST simplification
if (s2 != "") {
es2->emplace_back(i->first, new ExprString(std::move(s2)));
es2->emplace_back(i->first, new ExprString(alloc, s2));
}
};
for (; i != es.end(); ++i, --n) {

16
src/libexpr/include/nix/expr/primops.hh
View file

@ -8,22 +8,6 @@
namespace nix {
/**
* For functions where we do not expect deep recursion, we can use a sizable
* part of the stack a free allocation space.
*
* Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
*/
constexpr size_t nonRecursiveStackReservation = 128;
/**
* Functions that maybe applied to self-similar inputs, such as concatMap on a
* tree, should reserve a smaller part of the stack for allocation.
*
* Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes.
*/
constexpr size_t conservativeStackReservation = 16;
struct RegisterPrimOp
{
typedef std::vector<PrimOp> PrimOps;

2
src/libexpr/include/nix/expr/value.hh
View file

@ -155,7 +155,7 @@ class ListBuilder
Value * inlineElems[2] = {nullptr, nullptr};
public:
Value ** elems;
ListBuilder(EvalState & state, size_t size);
ListBuilder(size_t size);
// NOTE: Can be noexcept because we are just copying integral values and
// raw pointers.

4
src/libexpr/lexer-helpers.cc
View file

@ -1,11 +1,11 @@
#include "lexer-helpers.hh"
void nix::lexer::internal::initLoc(YYLTYPE * loc)
void nix::lexer::internal::initLoc(Parser::location_type * loc)
{
loc->beginOffset = loc->endOffset = 0;
}
void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, Parser::location_type * loc, const char * s, size_t len)
{
loc->stash();

10
src/libexpr/lexer-helpers.hh
View file

@ -2,16 +2,12 @@
#include <cstddef>
// including the generated headers twice leads to errors
#ifndef BISON_HEADER
# include "lexer-tab.hh"
# include "parser-tab.hh"
#endif
#include "parser-scanner-decls.hh"
namespace nix::lexer::internal {
void initLoc(YYLTYPE * loc);
void initLoc(Parser::location_type * loc);
void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
void adjustLoc(yyscan_t yyscanner, Parser::location_type * loc, const char * s, size_t len);
} // namespace nix::lexer::internal

4
src/libexpr/lexer.l
View file

@ -82,6 +82,10 @@ static void requireExperimentalFeature(const ExperimentalFeature & feature, cons
}
using enum nix::Parser::token::token_kind_type;
using YYSTYPE = nix::Parser::value_type;
using YYLTYPE = nix::Parser::location_type;
// yacc generates code that uses unannotated fallthrough.
#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"

5
src/libexpr/meson.build
View file

@ -69,6 +69,10 @@ if bdw_gc.found()
define_value = cxx.has_function(funcspec).to_int()
configdata_priv.set(define_name, define_value)
endforeach
if host_machine.system() == 'cygwin'
# undefined reference to `__wrap__Znwm'
configdata_pub.set('GC_NO_INLINE_STD_NEW', 1)
endif
endif
# Used in public header. Affects ABI!
configdata_pub.set('NIX_USE_BOEHMGC', bdw_gc.found().to_int())
@ -93,7 +97,6 @@ config_priv_h = configure_file(
)
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
parser_tab = custom_target(
input : 'parser.y',

12
src/libexpr/nixexpr.cc
View file

@ -11,7 +11,7 @@
namespace nix {
unsigned long Expr::nrExprs = 0;
Counter Expr::nrExprs;
ExprBlackHole eBlackHole;
@ -40,12 +40,12 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
{
printLiteralString(str, s);
printLiteralString(str, v.string_view());
}
void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const
{
str << s;
str << v.pathStr();
}
void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const
@ -57,7 +57,7 @@ void ExprSelect::show(const SymbolTable & symbols, std::ostream & str) const
{
str << "(";
e->show(symbols, str);
str << ")." << showAttrPath(symbols, attrPath);
str << ")." << showAttrPath(symbols, getAttrPath());
if (def) {
str << " or (";
def->show(symbols, str);
@ -261,7 +261,7 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const
str << "__curPos";
}
std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
std::string showAttrPath(const SymbolTable & symbols, std::span<const AttrName> attrPath)
{
std::ostringstream out;
bool first = true;
@ -362,7 +362,7 @@ void ExprSelect::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
e->bindVars(es, env);
if (def)
def->bindVars(es, env);
for (auto & i : attrPath)
for (auto & i : getAttrPath())
if (!i.symbol)
i.expr->bindVars(es, env);
}

17
src/libexpr/parser-scanner-decls.hh Normal file
View file

@ -0,0 +1,17 @@
#pragma once
#ifndef BISON_HEADER
# include "parser-tab.hh"
using YYSTYPE = nix::parser::BisonParser::value_type;
using YYLTYPE = nix::parser::BisonParser::location_type;
# include "lexer-tab.hh" // IWYU pragma: export
#endif
namespace nix {
class Parser : public parser::BisonParser
{
using BisonParser::BisonParser;
};
} // namespace nix

138
src/libexpr/parser.y
View file

@ -1,5 +1,7 @@
%skeleton "lalr1.cc"
%define api.location.type { ::nix::ParserLocation }
%define api.pure
%define api.namespace { ::nix::parser }
%define api.parser.class { BisonParser }
%locations
%define parse.error verbose
%defines
@ -26,19 +28,12 @@
#include "nix/expr/eval-settings.hh"
#include "nix/expr/parser-state.hh"
// Bison seems to have difficulty growing the parser stack when using C++ with
// a custom location type. This undocumented macro tells Bison that our
// location type is "trivially copyable" in C++-ese, so it is safe to use the
// same memcpy macro it uses to grow the stack that it uses with its own
// default location type. Without this, we get "error: memory exhausted" when
// parsing some large Nix files. Our other options are to increase the initial
// stack size (200 by default) to be as large as we ever want to support (so
// that growing the stack is unnecessary), or redefine the stack-relocation
// macro ourselves (which is also undocumented).
#define YYLTYPE_IS_TRIVIAL 1
#define YY_DECL int yylex \
(YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
#define YY_DECL \
int yylex( \
nix::Parser::value_type * yylval_param, \
nix::Parser::location_type * yylloc_param, \
yyscan_t yyscanner, \
nix::ParserState * state)
// For efficiency, we only track offsets; not line,column coordinates
# define YYLLOC_DEFAULT(Current, Rhs, N) \
@ -64,6 +59,7 @@ Expr * parseExprFromBuf(
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
std::pmr::polymorphic_allocator<char> & alloc,
SymbolTable & symbols,
const EvalSettings & settings,
PosTable & positions,
@ -78,24 +74,30 @@ Expr * parseExprFromBuf(
%{
#include "parser-tab.hh"
#include "lexer-tab.hh"
/* The parser is very performance sensitive and loses out on a lot
of performance even with basic stdlib assertions. Since those don't
affect ABI we can disable those just for this file. */
#if defined(_GLIBCXX_ASSERTIONS) && !defined(_GLIBCXX_DEBUG)
#undef _GLIBCXX_ASSERTIONS
#endif
#include "parser-scanner-decls.hh"
YY_DECL;
using namespace nix;
#define CUR_POS state->at(yyloc)
#define CUR_POS state->at(yylhs.location)
void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
void parser::BisonParser::error(const location_type &loc_, const std::string &error)
{
auto loc = loc_;
if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
loc->beginOffset = loc->endOffset;
loc.beginOffset = loc.endOffset;
}
throw ParseError({
.msg = HintFmt(error),
.pos = state->positions[state->at(*loc)]
.pos = state->positions[state->at(loc)]
});
}
@ -134,6 +136,7 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
std::vector<nix::AttrName> * attrNames;
std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
std::variant<nix::Expr *, std::string_view> * to_be_string;
std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
}
@ -148,7 +151,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
%type <inheritAttrs> attrs
%type <string_parts> string_parts_interpolated
%type <ind_string_parts> ind_string_parts
%type <e> path_start string_parts string_attr
%type <e> path_start
%type <to_be_string> string_parts string_attr
%type <id> attr
%token <id> ID
%token <str> STR IND_STR
@ -182,7 +186,7 @@ start: expr {
state->result = $1;
// This parser does not use yynerrs; suppress the warning.
(void) yynerrs;
(void) yynerrs_;
};
expr: expr_function;
@ -257,7 +261,7 @@ expr_op
| expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); }
| expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); }
| expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); }
| expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; }
| expr_op '?' attrpath { $$ = new ExprOpHasAttr(state->alloc, $1, std::move(*$3)); delete $3; }
| expr_op '+' expr_op
{ $$ = new ExprConcatStrings(state->at(@2), false, new std::vector<std::pair<PosIdx, Expr *> >({{state->at(@1), $1}, {state->at(@3), $3}})); }
| expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); }
@ -278,9 +282,9 @@ expr_app
expr_select
: expr_simple '.' attrpath
{ $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
| expr_simple '.' attrpath OR_KW expr_select
{ $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); }
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); }
| /* Backwards compatibility: because Nixpkgs has a function named ‘or’,
allow stuff like ‘map or [...]’. This production is problematic (see
https://github.com/NixOS/nix/issues/11118) and will be refactored in the
@ -303,7 +307,13 @@ expr_simple
}
| INT_LIT { $$ = new ExprInt($1); }
| FLOAT_LIT { $$ = new ExprFloat($1); }
| '"' string_parts '"' { $$ = $2; }
| '"' string_parts '"' {
std::visit(overloaded{
[&](std::string_view str) { $$ = new ExprString(state->alloc, str); },
[&](Expr * expr) { $$ = expr; }},
*$2);
delete $2;
}
| IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
$$ = state->stripIndentation(CUR_POS, std::move(*$2));
delete $2;
@ -314,11 +324,11 @@ expr_simple
$$ = new ExprConcatStrings(CUR_POS, false, $2);
}
| SPATH {
std::string path($1.p + 1, $1.l - 2);
std::string_view path($1.p + 1, $1.l - 2);
$$ = new ExprCall(CUR_POS,
new ExprVar(state->s.findFile),
{new ExprVar(state->s.nixPath),
new ExprString(std::move(path))});
new ExprString(state->alloc, path)});
}
| URI {
static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
@ -327,13 +337,13 @@ expr_simple
.msg = HintFmt("URL literals are disabled"),
.pos = state->positions[CUR_POS]
});
$$ = new ExprString(std::string($1));
$$ = new ExprString(state->alloc, $1);
}
| '(' expr ')' { $$ = $2; }
/* Let expressions `let {..., body = ...}' are just desugared
into `(rec {..., body = ...}).body'. */
| LET '{' binds '}'
{ $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); }
{ $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(state->alloc, noPos, $3, state->s.body); }
| REC '{' binds '}'
{ $3->recursive = true; $3->pos = CUR_POS; $$ = $3; }
| '{' binds1 '}'
@ -344,19 +354,19 @@ expr_simple
;
string_parts
: STR { $$ = new ExprString(std::string($1)); }
| string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); }
| { $$ = new ExprString(""); }
: STR { $$ = new std::variant<Expr *, std::string_view>($1); }
| string_parts_interpolated { $$ = new std::variant<Expr *, std::string_view>(new ExprConcatStrings(CUR_POS, true, $1)); }
| { $$ = new std::variant<Expr *, std::string_view>(std::string_view()); }
;
string_parts_interpolated
: string_parts_interpolated STR
{ $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); }
{ $$ = $1; $1->emplace_back(state->at(@2), new ExprString(state->alloc, $2)); }
| string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
| DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
| STR DOLLAR_CURLY expr '}' {
$$ = new std::vector<std::pair<PosIdx, Expr *>>;
$$->emplace_back(state->at(@1), new ExprString(std::string($1)));
$$->emplace_back(state->at(@1), new ExprString(state->alloc, $1));
$$->emplace_back(state->at(@2), $3);
}
;
@ -382,8 +392,8 @@ path_start
root filesystem accessor, rather than the accessor of the
current Nix expression. */
literal.front() == '/'
? new ExprPath(state->rootFS, std::move(path))
: new ExprPath(state->basePath.accessor, std::move(path));
? new ExprPath(state->alloc, state->rootFS, path)
: new ExprPath(state->alloc, state->basePath.accessor, path);
}
| HPATH {
if (state->settings.pureEval) {
@ -393,7 +403,7 @@ path_start
);
}
Path path(getHome() + std::string($1.p + 1, $1.l - 1));
$$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
$$ = new ExprPath(state->alloc, ref<SourceAccessor>(state->rootFS), path);
}
;
@ -437,7 +447,7 @@ binds1
$accum->attrs.emplace(
i.symbol,
ExprAttrs::AttrDef(
new ExprSelect(iPos, from, i.symbol),
new ExprSelect(state->alloc, iPos, from, i.symbol),
iPos,
ExprAttrs::AttrDef::Kind::InheritedFrom));
}
@ -454,15 +464,16 @@ attrs
: attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
| attrs string_attr
{ $$ = $1;
ExprString * str = dynamic_cast<ExprString *>($2);
if (str) {
$$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2));
delete str;
} else
throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in inherit"),
.pos = state->positions[state->at(@2)]
});
std::visit(overloaded {
[&](std::string_view str) { $$->emplace_back(AttrName(state->symbols.create(str)), state->at(@2)); },
[&](Expr * expr) {
throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in inherit"),
.pos = state->positions[state->at(@2)]
});
}
}, *$2);
delete $2;
}
| { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
;
@ -471,22 +482,20 @@ attrpath
: attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
| attrpath '.' string_attr
{ $$ = $1;
ExprString * str = dynamic_cast<ExprString *>($3);
if (str) {
$$->push_back(AttrName(state->symbols.create(str->s)));
delete str;
} else
$$->push_back(AttrName($3));
std::visit(overloaded {
[&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
[&](Expr * expr) { $$->push_back(AttrName(expr)); }
}, *$3);
delete $3;
}
| attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
| string_attr
{ $$ = new std::vector<AttrName>;
ExprString *str = dynamic_cast<ExprString *>($1);
if (str) {
$$->push_back(AttrName(state->symbols.create(str->s)));
delete str;
} else
$$->push_back(AttrName($1));
std::visit(overloaded {
[&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
[&](Expr * expr) { $$->push_back(AttrName(expr)); }
}, *$1);
delete $1;
}
;
@ -497,7 +506,7 @@ attr
string_attr
: '"' string_parts '"' { $$ = $2; }
| DOLLAR_CURLY expr '}' { $$ = $2; }
| DOLLAR_CURLY expr '}' { $$ = new std::variant<Expr *, std::string_view>($2); }
;
expr_list
@ -537,6 +546,7 @@ Expr * parseExprFromBuf(
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
std::pmr::polymorphic_allocator<char> & alloc,
SymbolTable & symbols,
const EvalSettings & settings,
PosTable & positions,
@ -551,6 +561,7 @@ Expr * parseExprFromBuf(
};
ParserState state {
.lexerState = lexerState,
.alloc = alloc,
.symbols = symbols,
.positions = positions,
.basePath = basePath,
@ -563,7 +574,8 @@ Expr * parseExprFromBuf(
Finally _destroy([&] { yylex_destroy(scanner); });
yy_scan_buffer(text, length, scanner);
yyparse(scanner, &state);
Parser parser(scanner, &state);
parser.parse();
return state.result;
}

25
src/libexpr/paths.cc
View file

@ -1,5 +1,7 @@
#include "nix/store/store-api.hh"
#include "nix/expr/eval.hh"
#include "nix/util/mounted-source-accessor.hh"
#include "nix/fetchers/fetch-to-store.hh"
namespace nix {
@ -18,4 +20,27 @@ SourcePath EvalState::storePath(const StorePath & path)
return {rootFS, CanonPath{store->printStorePath(path)}};
}
StorePath
EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
{
auto storePath = fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
allowPath(storePath); // FIXME: should just whitelist the entire virtual store
storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
auto narHash = store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
if (originalInput.getNarHash() && narHash != *originalInput.getNarHash())
throw Error(
(unsigned int) 102,
"NAR hash mismatch in input '%s', expected '%s' but got '%s'",
originalInput.to_string(),
narHash.to_string(HashFormat::SRI, true),
originalInput.getNarHash()->to_string(HashFormat::SRI, true));
return storePath;
}
} // namespace nix

11
src/libexpr/primops.cc
View file

@ -262,7 +262,7 @@ static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path,
{
state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport");
Env * env = &state.allocEnv(vScope->attrs()->size());
Env * env = &state.mem.allocEnv(vScope->attrs()->size());
env->up = &state.baseEnv;
auto staticEnv = std::make_shared<StaticEnv>(nullptr, state.staticBaseEnv, vScope->attrs()->size());
@ -1420,7 +1420,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
.debugThrow();
}
if (ingestionMethod == ContentAddressMethod::Raw::Text)
experimentalFeatureSettings.require(Xp::DynamicDerivations);
experimentalFeatureSettings.require(
Xp::DynamicDerivations, fmt("text-hashed derivation '%s', outputHashMode = \"text\"", drvName));
if (ingestionMethod == ContentAddressMethod::Raw::Git)
experimentalFeatureSettings.require(Xp::GitHashing);
};
@ -2412,7 +2413,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value ** args, Value
std::ostringstream out;
NixStringContext context;
printValueAsXML(state, true, false, *args[0], out, context, pos);
v.mkString(toView(out), context);
v.mkString(out.view(), context);
}
static RegisterPrimOp primop_toXML({
@ -2520,7 +2521,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value ** args, Valu
std::ostringstream out;
NixStringContext context;
printValueAsJSON(state, true, *args[0], pos, out, context);
v.mkString(toView(out), context);
v.mkString(out.view(), context);
}
static RegisterPrimOp primop_toJSON({
@ -3161,7 +3162,7 @@ static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value ** args,
// Step 1. Sort the name-value attrsets in place using the memory we allocate for the result
auto listView = args[0]->listView();
size_t listSize = listView.size();
auto & bindings = *state.allocBindings(listSize);
auto & bindings = *state.mem.allocBindings(listSize);
using ElemPtr = decltype(&bindings[0].value);
for (const auto & [n, v2] : enumerate(listView)) {

27
src/libexpr/primops/fetchTree.cc
View file

@ -10,6 +10,7 @@
#include "nix/util/url.hh"
#include "nix/expr/value-to-json.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/input-cache.hh"
#include <nlohmann/json.hpp>
@ -218,11 +219,11 @@ static void fetchTree(
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
}
auto [storePath, input2] = input.fetchToStore(state.store);
auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No);
state.allowPath(storePath);
auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false);
}
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, Value & v)
@ -561,14 +562,22 @@ static void fetch(
.hash = *expectedHash,
.references = {}});
if (state.store->isValidPath(expectedPath)) {
// Try to get the path from the local store or substituters
try {
state.store->ensurePath(expectedPath);
debug("using substituted/cached path '%s' for '%s'", state.store->printStorePath(expectedPath), *url);
state.allowAndSetStorePathString(expectedPath, v);
return;
} catch (Error & e) {
debug(
"substitution of '%s' failed, will try to download: %s",
state.store->printStorePath(expectedPath),
e.what());
// Fall through to download
}
}
// TODO: fetching may fail, yet the path may be substitutable.
// https://github.com/NixOS/nix/issues/4313
// Download the file/tarball if substitution failed or no hash was provided
auto storePath = unpack ? fetchToStore(
state.fetchSettings,
*state.store,
@ -579,7 +588,11 @@ static void fetch(
if (expectedHash) {
auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash
: hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath));
: hashPath(
{state.store->requireStoreObjectAccessor(storePath)},
FileSerialisationMethod::Flat,
HashAlgorithm::SHA256)
.hash;
if (hash != *expectedHash) {
state
.error<EvalError>(

2
src/libexpr/primops/fromTOML.cc
View file

@ -139,7 +139,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
attrs.alloc("_type").mkStringNoCopy("timestamp");
std::ostringstream s;
s << t;
auto str = toView(s);
auto str = s.view();
forceNoNullByte(str);
attrs.alloc("value").mkString(str);
v.mkAttrs(attrs);

2
src/libexpr/print.cc
View file

@ -461,7 +461,7 @@ private:
std::ostringstream s;
s << state.positions[v.lambda().fun->pos];
output << " @ " << filterANSIEscapes(toView(s));
output << " @ " << filterANSIEscapes(s.view());
}
} else if (v.isPrimOp()) {
if (v.primOp())

1
src/libfetchers-c/meson.build
View file

@ -32,7 +32,6 @@ add_project_arguments(
)
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'nix_api_fetchers.cc',

3
src/libfetchers-tests/meson.build
View file

@ -37,7 +37,6 @@ libgit2 = dependency('libgit2')
deps_private += libgit2
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'access-tokens.cc',
@ -64,7 +63,7 @@ this_exe = executable(
test(
meson.project_name(),
this_exe,
env : asan_test_options_env + {
env : {
'_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data',
},
protocol : 'gtest',

1
src/libfetchers-tests/package.nix
View file

@ -61,7 +61,6 @@ mkMesonExecutable (finalAttrs: {
buildInputs = [ writableTmpDirAsHomeHook ];
}
''
export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0
export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
touch $out

55
src/libfetchers-tests/public-key.cc
View file

@ -1,14 +1,14 @@
#include <gtest/gtest.h>
#include "nix/fetchers/fetchers.hh"
#include "nix/util/json-utils.hh"
#include <nlohmann/json.hpp>
#include "nix/util/tests/characterization.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
using nlohmann::json;
class PublicKeyTest : public CharacterizationTest
class PublicKeyTest : public JsonCharacterizationTest<fetchers::PublicKey>,
public ::testing::WithParamInterface<std::pair<std::string_view, fetchers::PublicKey>>
{
std::filesystem::path unitTestData = getUnitTestData() / "public-key";
@ -19,30 +19,35 @@ public:
}
};
#define TEST_JSON(FIXTURE, NAME, VAL) \
TEST_F(FIXTURE, PublicKey_##NAME##_from_json) \
{ \
readTest(#NAME ".json", [&](const auto & encoded_) { \
fetchers::PublicKey expected{VAL}; \
fetchers::PublicKey got = nlohmann::json::parse(encoded_); \
ASSERT_EQ(got, expected); \
}); \
} \
\
TEST_F(FIXTURE, PublicKey_##NAME##_to_json) \
{ \
writeTest( \
#NAME ".json", \
[&]() -> json { return nlohmann::json(fetchers::PublicKey{VAL}); }, \
[](const auto & file) { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
}
TEST_P(PublicKeyTest, from_json)
{
const auto & [name, expected] = GetParam();
readJsonTest(name, expected);
}
TEST_JSON(PublicKeyTest, simple, (fetchers::PublicKey{.type = "ssh-rsa", .key = "ABCDE"}))
TEST_P(PublicKeyTest, to_json)
{
const auto & [name, value] = GetParam();
writeJsonTest(name, value);
}
TEST_JSON(PublicKeyTest, defaultType, fetchers::PublicKey{.key = "ABCDE"})
#undef TEST_JSON
INSTANTIATE_TEST_SUITE_P(
PublicKeyJSON,
PublicKeyTest,
::testing::Values(
std::pair{
"simple",
fetchers::PublicKey{
.type = "ssh-rsa",
.key = "ABCDE",
},
},
std::pair{
"defaultType",
fetchers::PublicKey{
.key = "ABCDE",
},
}));
TEST_F(PublicKeyTest, PublicKey_noRoundTrip_from_json)
{

15
src/libfetchers/fetch-to-store.cc
View file

@ -1,6 +1,7 @@
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/fetchers.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/util/environment-variables.hh"
namespace nix {
@ -27,14 +28,22 @@ StorePath fetchToStore(
std::optional<fetchers::Cache::Key> cacheKey;
if (!filter && path.accessor->fingerprint) {
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
auto [subpath, fingerprint] = filter ? std::pair<CanonPath, std::optional<std::string>>{path.path, std::nullopt}
: path.accessor->getFingerprint(path.path);
if (fingerprint) {
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *fingerprint, method, subpath.abs());
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
debug("store path cache hit for '%s'", path);
return res->storePath;
}
} else
} else {
static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
if (barf && !filter)
throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter);
// FIXME: could still provide in-memory caching keyed on `SourcePath`.
debug("source path '%s' is uncacheable", path);
}
Activity act(
*logger,

21
src/libfetchers/fetchers.cc
View file

@ -3,8 +3,8 @@
#include "nix/util/source-path.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/json-utils.hh"
#include "nix/fetchers/store-path-accessor.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include <nlohmann/json.hpp>
@ -332,10 +332,19 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
auto accessor = makeStorePathAccessor(store, storePath);
auto accessor = store->requireStoreObjectAccessor(storePath);
accessor->fingerprint = getFingerprint(store);
// Store a cache entry for the substituted tree so later fetches
// can reuse the existing nar instead of copying the unpacked
// input back into the store on every evaluation.
if (accessor->fingerprint) {
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
settings->getCache()->upsert(cacheKey, *store, {}, storePath);
}
accessor->setPathDisplay("«" + to_string() + "»");
return {accessor, *this};
@ -346,8 +355,10 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
auto [accessor, result] = scheme->getAccessor(store, *this);
assert(!accessor->fingerprint);
accessor->fingerprint = result.getFingerprint(store);
if (!accessor->fingerprint)
accessor->fingerprint = result.getFingerprint(store);
else
result.cachedFingerprint = accessor->fingerprint;
return {accessor, std::move(result)};
}
@ -509,7 +520,7 @@ fetchers::PublicKey adl_serializer<fetchers::PublicKey>::from_json(const json &
return res;
}
void adl_serializer<fetchers::PublicKey>::to_json(json & json, fetchers::PublicKey p)
void adl_serializer<fetchers::PublicKey>::to_json(json & json, const fetchers::PublicKey & p)
{
json["type"] = p.type;
json["key"] = p.key;

26
src/libfetchers/filtering-source-accessor.cc
View file

@ -16,15 +16,26 @@ std::string FilteringSourceAccessor::readFile(const CanonPath & path)
return next->readFile(prefix / path);
}
void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
{
checkAccess(path);
return next->readFile(prefix / path, sink, sizeCallback);
}
bool FilteringSourceAccessor::pathExists(const CanonPath & path)
{
return isAllowed(path) && next->pathExists(prefix / path);
}
std::optional<SourceAccessor::Stat> FilteringSourceAccessor::maybeLstat(const CanonPath & path)
{
return isAllowed(path) ? next->maybeLstat(prefix / path) : std::nullopt;
}
SourceAccessor::Stat FilteringSourceAccessor::lstat(const CanonPath & path)
{
checkAccess(path);
return next->maybeLstat(prefix / path);
return next->lstat(prefix / path);
}
SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path)
@ -49,6 +60,13 @@ std::string FilteringSourceAccessor::showPath(const CanonPath & path)
return displayPrefix + next->showPath(prefix / path) + displaySuffix;
}
std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFingerprint(const CanonPath & path)
{
if (fingerprint)
return {path, fingerprint};
return next->getFingerprint(prefix / path);
}
void FilteringSourceAccessor::checkAccess(const CanonPath & path)
{
if (!isAllowed(path))
@ -59,12 +77,12 @@ void FilteringSourceAccessor::checkAccess(const CanonPath & path)
struct AllowListSourceAccessorImpl : AllowListSourceAccessor
{
std::set<CanonPath> allowedPrefixes;
boost::unordered_flat_set<CanonPath, std::hash<CanonPath>> allowedPaths;
boost::unordered_flat_set<CanonPath> allowedPaths;
AllowListSourceAccessorImpl(
ref<SourceAccessor> next,
std::set<CanonPath> && allowedPrefixes,
boost::unordered_flat_set<CanonPath, std::hash<CanonPath>> && allowedPaths,
boost::unordered_flat_set<CanonPath> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
: AllowListSourceAccessor(SourcePath(next), std::move(makeNotAllowedError))
, allowedPrefixes(std::move(allowedPrefixes))
@ -86,7 +104,7 @@ struct AllowListSourceAccessorImpl : AllowListSourceAccessor
ref<AllowListSourceAccessor> AllowListSourceAccessor::create(
ref<SourceAccessor> next,
std::set<CanonPath> && allowedPrefixes,
boost::unordered_flat_set<CanonPath, std::hash<CanonPath>> && allowedPaths,
boost::unordered_flat_set<CanonPath> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
{
return make_ref<AllowListSourceAccessorImpl>(

4
src/libfetchers/git-utils.cc
View file

@ -817,7 +817,7 @@ struct GitSourceAccessor : SourceAccessor
return toHash(*git_tree_entry_id(entry));
}
boost::unordered_flat_map<CanonPath, TreeEntry, std::hash<CanonPath>> lookupCache;
boost::unordered_flat_map<CanonPath, TreeEntry> lookupCache;
/* Recursively look up 'path' relative to the root. */
git_tree_entry * lookup(State & state, const CanonPath & path)
@ -1254,7 +1254,7 @@ GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllow
makeFSSourceAccessor(path),
std::set<CanonPath>{wd.files},
// Always allow access to the root, but not its children.
boost::unordered_flat_set<CanonPath, std::hash<CanonPath>>{CanonPath::root},
boost::unordered_flat_set<CanonPath>{CanonPath::root},
std::move(makeNotAllowedError))
.cast<SourceAccessor>();
if (exportIgnore)

6
src/libfetchers/git.cc
View file

@ -15,6 +15,7 @@
#include "nix/fetchers/fetch-settings.hh"
#include "nix/util/json-utils.hh"
#include "nix/util/archive.hh"
#include "nix/util/mounted-source-accessor.hh"
#include <regex>
#include <string.h>
@ -892,8 +893,7 @@ struct GitInputScheme : InputScheme
return makeFingerprint(*rev);
else {
auto repoInfo = getRepoInfo(input);
if (auto repoPath = repoInfo.getPath();
repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) {
if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) {
/* Calculate a fingerprint that takes into account the
deleted and modified/added files. */
HashSink hashSink{HashAlgorithm::SHA512};
@ -906,7 +906,7 @@ struct GitInputScheme : InputScheme
writeString("deleted:", hashSink);
writeString(file.abs(), hashSink);
}
return makeFingerprint(*repoInfo.workdirInfo.headRev)
return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev))
+ ";d=" + hashSink.finish().hash.to_string(HashFormat::Base16, false);
}
return std::nullopt;

22
src/libfetchers/github.cc
View file

@ -398,8 +398,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
auto json = nlohmann::json::parse(
readFile(store->toRealPath(downloadFile(store, *input.settings, url, "source", headers).storePath)));
store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
return RefInfo{
.rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1),
@ -472,8 +473,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
auto json = nlohmann::json::parse(
readFile(store->toRealPath(downloadFile(store, *input.settings, url, "source", headers).storePath)));
store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)};
@ -548,13 +550,10 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::string refUri;
if (ref == "HEAD") {
auto file = store->toRealPath(
downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath);
std::ifstream is(file);
std::string line;
getline(is, line);
auto downloadFileResult = downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers);
auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
auto remoteLine = git::parseLsRemoteLine(line);
auto remoteLine = git::parseLsRemoteLine(getLine(contents).first);
if (!remoteLine) {
throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref);
}
@ -564,9 +563,10 @@ struct SourceHutInputScheme : GitArchiveInputScheme
}
std::regex refRegex(refUri);
auto file = store->toRealPath(
downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath);
std::ifstream is(file);
auto downloadFileResult =
downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers);
auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
std::istringstream is(contents);
std::string line;
std::optional<std::string> id;

8
src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh
View file

@ -36,8 +36,12 @@ struct FilteringSourceAccessor : SourceAccessor
std::string readFile(const CanonPath & path) override;
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
bool pathExists(const CanonPath & path) override;
Stat lstat(const CanonPath & path) override;
std::optional<Stat> maybeLstat(const CanonPath & path) override;
DirEntries readDirectory(const CanonPath & path) override;
@ -46,6 +50,8 @@ struct FilteringSourceAccessor : SourceAccessor
std::string showPath(const CanonPath & path) override;
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
/**
* Call `makeNotAllowedError` to throw a `RestrictedPathError`
* exception if `isAllowed()` returns `false` for `path`.
@ -72,7 +78,7 @@ struct AllowListSourceAccessor : public FilteringSourceAccessor
static ref<AllowListSourceAccessor> create(
ref<SourceAccessor> next,
std::set<CanonPath> && allowedPrefixes,
boost::unordered_flat_set<CanonPath, std::hash<CanonPath>> && allowedPaths,
boost::unordered_flat_set<CanonPath> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError);
using FilteringSourceAccessor::FilteringSourceAccessor;

1
src/libfetchers/include/nix/fetchers/meson.build
View file

@ -11,6 +11,5 @@ headers = files(
'git-utils.hh',
'input-cache.hh',
'registry.hh',
'store-path-accessor.hh',
'tarball.hh',
)

14
src/libfetchers/include/nix/fetchers/store-path-accessor.hh
View file

@ -1,14 +0,0 @@
#pragma once
#include "nix/util/source-path.hh"
namespace nix {
class StorePath;
class Store;
ref<SourceAccessor> makeStorePathAccessor(ref<Store> store, const StorePath & storePath);
SourcePath getUnfilteredRootPath(CanonPath path);
} // namespace nix

4
src/libfetchers/mercurial.cc
View file

@ -6,7 +6,6 @@
#include "nix/util/tarfile.hh"
#include "nix/store/store-api.hh"
#include "nix/util/url-parts.hh"
#include "nix/fetchers/store-path-accessor.hh"
#include "nix/fetchers/fetch-settings.hh"
#include <sys/time.h>
@ -330,8 +329,7 @@ struct MercurialInputScheme : InputScheme
Input input(_input);
auto storePath = fetchToStore(store, input);
auto accessor = makeStorePathAccessor(store, storePath);
auto accessor = store->requireStoreObjectAccessor(storePath);
accessor->setPathDisplay("«" + input.to_string() + "»");

2
src/libfetchers/meson.build
View file

@ -32,7 +32,6 @@ libgit2 = dependency('libgit2', version : '>= 1.9')
deps_private += libgit2
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'attrs.cc',
@ -50,7 +49,6 @@ sources = files(
'mercurial.cc',
'path.cc',
'registry.cc',
'store-path-accessor.cc',
'tarball.cc',
)

43
src/libfetchers/path.cc
View file

@ -1,7 +1,6 @@
#include "nix/fetchers/fetchers.hh"
#include "nix/store/store-api.hh"
#include "nix/util/archive.hh"
#include "nix/fetchers/store-path-accessor.hh"
#include "nix/fetchers/cache.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/fetch-settings.hh"
@ -124,8 +123,6 @@ struct PathInputScheme : InputScheme
auto absPath = getAbsPath(input);
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
// FIXME: check whether access to 'path' is allowed.
auto storePath = store->maybeParseStorePath(absPath.string());
@ -134,43 +131,33 @@ struct PathInputScheme : InputScheme
time_t mtime = 0;
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
// FIXME: try to substitute storePath.
auto src = sinkToSource(
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
storePath = store->addToStoreFromDump(*src, "source");
}
// To avoid copying the path again to the /nix/store, we need to add a cache entry.
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto fp = getFingerprint(store, input);
if (fp) {
auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/");
input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath);
}
auto accessor = store->requireStoreObjectAccessor(*storePath);
// To prevent `fetchToStore()` copying the path again to Nix
// store, pre-create an entry in the fetcher cache.
auto info = store->queryPathInfo(*storePath);
accessor->fingerprint =
fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
input.settings->getCache()->upsert(
makeFetchToStoreCacheKey(
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
*store,
{},
*storePath);
/* Trust the lastModified value supplied by the user, if
any. It's not a "secure" attribute so we don't care. */
if (!input.getLastModified())
input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
return {makeStorePathAccessor(store, *storePath), std::move(input)};
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (isRelative(input))
return std::nullopt;
/* If this path is in the Nix store, use the hash of the
store object and the subpath. */
auto path = getAbsPath(input);
try {
auto [storePath, subPath] = store->toStorePath(path.string());
auto info = store->queryPathInfo(storePath);
return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath);
} catch (Error &) {
return std::nullopt;
}
return {accessor, std::move(input)};
}
std::optional<ExperimentalFeature> experimentalFeature() const override

11
src/libfetchers/store-path-accessor.cc
View file

@ -1,11 +0,0 @@
#include "nix/fetchers/store-path-accessor.hh"
#include "nix/store/store-api.hh"
namespace nix {
ref<SourceAccessor> makeStorePathAccessor(ref<Store> store, const StorePath & storePath)
{
return projectSubdirSourceAccessor(store->getFSAccessor(), storePath.to_string());
}
} // namespace nix

15
src/libfetchers/tarball.cc
View file

@ -6,7 +6,6 @@
#include "nix/util/archive.hh"
#include "nix/util/tarfile.hh"
#include "nix/util/types.hh"
#include "nix/fetchers/store-path-accessor.hh"
#include "nix/store/store-api.hh"
#include "nix/fetchers/git-utils.hh"
#include "nix/fetchers/fetch-settings.hh"
@ -43,7 +42,7 @@ DownloadFileResult downloadFile(
if (cached && !cached->expired)
return useCached();
FileTransferRequest request(ValidURL{url});
FileTransferRequest request(VerbatimURL{url});
request.headers = headers;
if (cached)
request.expectedETag = getStrAttr(cached->value, "etag");
@ -108,13 +107,13 @@ DownloadFileResult downloadFile(
static DownloadTarballResult downloadTarball_(
const Settings & settings, const std::string & urlS, const Headers & headers, const std::string & displayPrefix)
{
ValidURL url = urlS;
ParsedURL url = parseURL(urlS);
// Some friendly error messages for common mistakes.
// Namely lets catch when the url is a local file path, but
// it is not in fact a tarball.
if (url.scheme() == "file") {
std::filesystem::path localPath = renderUrlPathEnsureLegal(url.path());
if (url.scheme == "file") {
std::filesystem::path localPath = renderUrlPathEnsureLegal(url.path);
if (!exists(localPath)) {
throw Error("tarball '%s' does not exist.", localPath);
}
@ -165,7 +164,7 @@ static DownloadTarballResult downloadTarball_(
/* Note: if the download is cached, `importTarball()` will receive
no data, which causes it to import an empty tarball. */
auto archive = !url.path().empty() && hasSuffix(toLower(url.path().back()), ".zip") ? ({
auto archive = !url.path.empty() && hasSuffix(toLower(url.path.back()), ".zip") ? ({
/* In streaming mode, libarchive doesn't handle
symlinks in zip files correctly (#10649). So write
the entire file to disk so libarchive can access it
@ -179,7 +178,7 @@ static DownloadTarballResult downloadTarball_(
}
TarArchive{path};
})
: TarArchive{*source};
: TarArchive{*source};
auto tarballCache = getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
@ -354,7 +353,7 @@ struct FileInputScheme : CurlInputScheme
auto narHash = store->queryPathInfo(file.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
auto accessor = makeStorePathAccessor(store, file.storePath);
auto accessor = ref{store->getFSAccessor(file.storePath)};
accessor->setPathDisplay("«" + input.to_string() + "»");

1
src/libflake-c/meson.build
View file

@ -32,7 +32,6 @@ deps_public_maybe_subproject = [
subdir('nix-meson-build-support/subprojects')
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'nix_api_flake.cc',

7
src/libflake-tests/flakeref.cc
View file

@ -1,8 +1,15 @@
#include <gtest/gtest.h>
#include <set>
#include <string>
#include <utility>
#include "nix/fetchers/fetch-settings.hh"
#include "nix/flake/flakeref.hh"
#include "nix/fetchers/attrs.hh"
#include "nix/fetchers/fetchers.hh"
#include "nix/util/configuration.hh"
#include "nix/util/error.hh"
#include "nix/util/experimental-features.hh"
namespace nix {

3
src/libflake-tests/meson.build
View file

@ -34,7 +34,6 @@ gtest = dependency('gtest', main : true)
deps_private += gtest
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'flakeref.cc',
@ -59,7 +58,7 @@ this_exe = executable(
test(
meson.project_name(),
this_exe,
env : asan_test_options_env + {
env : {
'_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data',
'NIX_CONFIG' : 'extra-experimental-features = flakes',
'HOME' : meson.current_build_dir() / 'test-home',

12
src/libflake-tests/nix_api_flake.cc
View file

@ -1,15 +1,17 @@
#include <gtest/gtest.h>
#include <filesystem>
#include <string>
#include "nix/util/file-system.hh"
#include "nix_api_store.h"
#include "nix_api_util.h"
#include "nix_api_expr.h"
#include "nix_api_value.h"
#include "nix_api_flake.h"
#include "nix/expr/tests/nix_api_expr.hh"
#include "nix/util/tests/string_callback.hh"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "nix/store/tests/nix_api_store.hh"
#include "nix/util/tests/nix_api_util.hh"
#include "nix_api_fetchers.h"
namespace nixC {

1
src/libflake-tests/package.nix
View file

@ -59,7 +59,6 @@ mkMesonExecutable (finalAttrs: {
buildInputs = [ writableTmpDirAsHomeHook ];
}
(''
export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0
export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
export NIX_CONFIG="extra-experimental-features = flakes"
${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}

4
src/libflake-tests/url-name.cc
View file

@ -1,6 +1,8 @@
#include "nix/flake/url-name.hh"
#include <gtest/gtest.h>
#include "nix/flake/url-name.hh"
#include "nix/util/url.hh"
namespace nix {
/* ----------- tests for url-name.hh --------------------------------------------------*/

25
src/libflake/config.cc
View file

@ -1,9 +1,30 @@
#include <nlohmann/json.hpp>
#include <assert.h>
#include <stdint.h>
#include <nlohmann/json_fwd.hpp>
#include <cctype>
#include <filesystem>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <variant>
#include <vector>
#include <format>
#include "nix/util/users.hh"
#include "nix/util/config-global.hh"
#include "nix/flake/settings.hh"
#include "nix/flake/flake.hh"
#include <nlohmann/json.hpp>
#include "nix/util/ansicolor.hh"
#include "nix/util/configuration.hh"
#include "nix/util/file-system.hh"
#include "nix/util/fmt.hh"
#include "nix/util/logging.hh"
#include "nix/util/strings.hh"
#include "nix/util/types.hh"
#include "nix/util/util.hh"
namespace nix::flake {

26
src/libflake/flake-primops.cc
View file

@ -1,8 +1,34 @@
#include <stdint.h>
#include <functional>
#include <map>
#include <optional>
#include <set>
#include <string>
#include <utility>
#include <variant>
#include <vector>
#include "nix/flake/flake-primops.hh"
#include "nix/expr/eval.hh"
#include "nix/flake/flake.hh"
#include "nix/flake/flakeref.hh"
#include "nix/flake/settings.hh"
#include "nix/expr/attr-set.hh"
#include "nix/expr/eval-error.hh"
#include "nix/expr/eval-inline.hh"
#include "nix/expr/eval-settings.hh"
#include "nix/expr/symbol-table.hh"
#include "nix/expr/value.hh"
#include "nix/fetchers/attrs.hh"
#include "nix/fetchers/fetchers.hh"
#include "nix/util/configuration.hh"
#include "nix/util/error.hh"
#include "nix/util/experimental-features.hh"
#include "nix/util/pos-idx.hh"
#include "nix/util/pos-table.hh"
#include "nix/util/source-path.hh"
#include "nix/util/types.hh"
#include "nix/util/util.hh"
namespace nix::flake::primops {

129
src/libflake/flake.cc
View file

@ -1,9 +1,32 @@
#include <nlohmann/json.hpp>
#include <assert.h>
#include <stdint.h>
#include <boost/container/detail/std_fwd.hpp>
#include <boost/core/pointer_traits.hpp>
#include <boost/unordered/detail/foa/table.hpp>
#include <algorithm>
#include <filesystem>
#include <functional>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <span>
#include <string>
#include <tuple>
#include <utility>
#include <variant>
#include <vector>
#include <format>
#include "nix/util/terminal.hh"
#include "nix/util/ref.hh"
#include "nix/util/environment-variables.hh"
#include "nix/flake/flake.hh"
#include "nix/expr/eval.hh"
#include "nix/expr/eval-cache.hh"
#include "nix/expr/eval-settings.hh"
#include "nix/flake/lockfile.hh"
#include "nix/expr/primops.hh"
#include "nix/expr/eval-inline.hh"
#include "nix/store/store-api.hh"
#include "nix/fetchers/fetchers.hh"
@ -11,34 +34,41 @@
#include "nix/fetchers/fetch-settings.hh"
#include "nix/flake/settings.hh"
#include "nix/expr/value-to-json.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/memory-source-accessor.hh"
#include "nix/fetchers/input-cache.hh"
#include <nlohmann/json.hpp>
#include "nix/expr/attr-set.hh"
#include "nix/expr/eval-error.hh"
#include "nix/expr/nixexpr.hh"
#include "nix/expr/symbol-table.hh"
#include "nix/expr/value.hh"
#include "nix/expr/value/context.hh"
#include "nix/fetchers/attrs.hh"
#include "nix/fetchers/registry.hh"
#include "nix/flake/flakeref.hh"
#include "nix/store/path.hh"
#include "nix/util/canon-path.hh"
#include "nix/util/configuration.hh"
#include "nix/util/error.hh"
#include "nix/util/experimental-features.hh"
#include "nix/util/file-system.hh"
#include "nix/util/fmt.hh"
#include "nix/util/hash.hh"
#include "nix/util/logging.hh"
#include "nix/util/pos-idx.hh"
#include "nix/util/pos-table.hh"
#include "nix/util/position.hh"
#include "nix/util/source-path.hh"
#include "nix/util/types.hh"
#include "nix/util/util.hh"
namespace nix {
struct SourceAccessor;
using namespace flake;
namespace flake {
static StorePath copyInputToStore(
EvalState & state, fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor)
{
auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName());
state.allowPath(storePath);
auto narHash = state.store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
return storePath;
}
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
{
if (value.isThunk() && value.isTrivial())
@ -360,11 +390,14 @@ static Flake getFlake(
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
}
// Copy the tree to the store.
auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, cachedInput.accessor);
// Re-parse flake.nix from the store.
return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath);
return readFlake(
state,
originalRef,
resolvedRef,
lockedRef,
state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)),
lockRootAttrPath);
}
Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries)
@ -721,11 +754,10 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
// FIXME: allow input to be lazy.
auto storePath = copyInputToStore(
state, lockedRef.input, input.ref->input, cachedInput.accessor);
return {state.storePath(storePath), lockedRef};
return {
state.storePath(
state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor)),
lockedRef};
}
}();
@ -875,7 +907,7 @@ static ref<SourceAccessor> makeInternalFS()
internalFS->setPathDisplay("«flakes-internal»", "");
internalFS->addFile(
CanonPath("call-flake.nix"),
#include "call-flake.nix.gen.hh"
#include "call-flake.nix.gen.hh" // IWYU pragma: keep
);
return internalFS;
}
@ -937,8 +969,6 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
state.callFunction(*vCallFlake, args, vRes, noPos);
}
} // namespace flake
std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const
{
if (lockFile.isUnlocked(fetchSettings))
@ -966,4 +996,41 @@ std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store, const f
Flake::~Flake() {}
ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake)
{
auto fingerprint = state.settings.useEvalCache && state.settings.pureEval
? lockedFlake->getFingerprint(state.store, state.fetchSettings)
: std::nullopt;
auto rootLoader = [&state, lockedFlake]() {
/* For testing whether the evaluation cache is
complete. */
if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
throw Error("not everything is cached, but evaluation is not allowed");
auto vFlake = state.allocValue();
callFlake(state, *lockedFlake, *vFlake);
state.forceAttrs(*vFlake, noPos, "while parsing cached flake data");
auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs"));
assert(aOutputs);
return aOutputs->value;
};
if (fingerprint) {
auto search = state.evalCaches.find(fingerprint.value());
if (search == state.evalCaches.end()) {
search = state.evalCaches
.emplace(fingerprint.value(), make_ref<eval_cache::EvalCache>(fingerprint, state, rootLoader))
.first;
}
return search->second;
} else {
return make_ref<eval_cache::EvalCache>(std::nullopt, state, rootLoader);
}
}
} // namespace flake
} // namespace nix

31
src/libflake/flakeref.cc
View file

@ -1,10 +1,39 @@
#include <assert.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <filesystem>
#include <ostream>
#include <string_view>
#include <vector>
#include <optional>
#include <regex>
#include <string>
#include <tuple>
#include <utility>
#include "nix/flake/flakeref.hh"
#include "nix/store/store-api.hh"
#include "nix/util/url.hh"
#include "nix/util/url-parts.hh"
#include "nix/fetchers/fetchers.hh"
#include "nix/util/error.hh"
#include "nix/util/file-system.hh"
#include "nix/util/fmt.hh"
#include "nix/util/logging.hh"
#include "nix/util/strings.hh"
#include "nix/util/util.hh"
#include "nix/fetchers/attrs.hh"
#include "nix/fetchers/registry.hh"
#include "nix/store/outputs-spec.hh"
#include "nix/util/ref.hh"
#include "nix/util/types.hh"
namespace nix {
class Store;
struct SourceAccessor;
namespace fetchers {
struct Settings;
} // namespace fetchers
#if 0
// 'dir' path elements cannot start with a '.'. We also reject

7
src/libflake/include/nix/flake/flake-primops.hh
View file

@ -1,7 +1,12 @@
#pragma once
#include "nix/expr/eval.hh"
#include "nix/flake/settings.hh"
namespace nix {
namespace flake {
struct Settings;
} // namespace flake
} // namespace nix
namespace nix::flake::primops {

6
src/libflake/include/nix/flake/flake.hh
View file

@ -5,6 +5,7 @@
#include "nix/flake/flakeref.hh"
#include "nix/flake/lockfile.hh"
#include "nix/expr/value.hh"
#include "nix/expr/eval-cache.hh"
namespace nix {
@ -218,6 +219,11 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & flakeRe
void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v);
/**
* Open an evaluation cache for a flake.
*/
ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake);
} // namespace flake
void emitTreeAttrs(

10
src/libflake/include/nix/flake/flakeref.hh
View file

@ -2,9 +2,11 @@
///@file
#include <regex>
#include <iosfwd>
#include <string>
#include <tuple>
#include <utility>
#include "nix/util/types.hh"
#include "nix/fetchers/fetchers.hh"
#include "nix/store/outputs-spec.hh"
#include "nix/fetchers/registry.hh"
@ -12,6 +14,10 @@ namespace nix {
class Store;
namespace fetchers {
struct Settings;
} // namespace fetchers
typedef std::string FlakeId;
/**

5
src/libflake/include/nix/flake/settings.hh
View file

@ -1,9 +1,10 @@
#pragma once
///@file
#include "nix/util/configuration.hh"
#include <sys/types.h>
#include <string>
#include "nix/util/configuration.hh"
namespace nix {
// Forward declarations

7
src/libflake/include/nix/flake/url-name.hh
View file

@ -1,9 +1,8 @@
#include "nix/util/url.hh"
#include "nix/util/url-parts.hh"
#include "nix/util/util.hh"
#include "nix/util/split.hh"
#include <optional>
#include <string>
namespace nix {
struct ParsedURL;
/**
* Try to extract a reasonably unique and meaningful, human-readable

52
src/libflake/lockfile.cc
View file

@ -1,15 +1,49 @@
#include "nix/fetchers/fetch-settings.hh"
#include "nix/flake/settings.hh"
#include "nix/flake/lockfile.hh"
#include "nix/store/store-api.hh"
#include "nix/util/strings.hh"
#include <boost/unordered/unordered_flat_set.hpp>
#include <nlohmann/json.hpp>
#include <assert.h>
#include <boost/unordered/unordered_flat_set_fwd.hpp>
#include <nlohmann/detail/iterators/iter_impl.hpp>
#include <nlohmann/detail/iterators/iteration_proxy.hpp>
#include <nlohmann/json_fwd.hpp>
#include <algorithm>
#include <iomanip>
#include <boost/unordered/unordered_flat_set.hpp>
#include <iterator>
#include <nlohmann/json.hpp>
#include <compare>
#include <ctime>
#include <format>
#include <functional>
#include <map>
#include <memory>
#include <optional>
#include <ostream>
#include <regex>
#include <set>
#include <string>
#include <string_view>
#include <utility>
#include <variant>
#include <vector>
#include "nix/fetchers/fetch-settings.hh"
#include "nix/flake/lockfile.hh"
#include "nix/util/strings.hh"
#include "nix/fetchers/attrs.hh"
#include "nix/fetchers/fetchers.hh"
#include "nix/flake/flakeref.hh"
#include "nix/store/path.hh"
#include "nix/util/ansicolor.hh"
#include "nix/util/configuration.hh"
#include "nix/util/error.hh"
#include "nix/util/fmt.hh"
#include "nix/util/hash.hh"
#include "nix/util/logging.hh"
#include "nix/util/ref.hh"
#include "nix/util/types.hh"
#include "nix/util/util.hh"
namespace nix {
class Store;
} // namespace nix
namespace nix::flake {

1
src/libflake/meson.build
View file

@ -29,7 +29,6 @@ nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
deps_public += nlohmann_json
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
subdir('nix-meson-build-support/generate-header')

4
src/libflake/settings.cc
View file

@ -1,5 +1,9 @@
#include <vector>
#include "nix/flake/settings.hh"
#include "nix/flake/flake-primops.hh"
#include "nix/expr/eval-settings.hh"
#include "nix/expr/eval.hh"
namespace nix::flake {

8
src/libflake/url-name.cc
View file

@ -1,6 +1,10 @@
#include "nix/flake/url-name.hh"
#include <regex>
#include <iostream>
#include <optional>
#include <string>
#include "nix/flake/url-name.hh"
#include "nix/util/strings.hh"
#include "nix/util/url.hh"
namespace nix {

1
src/libmain-c/meson.build
View file

@ -28,7 +28,6 @@ deps_public_maybe_subproject = [
subdir('nix-meson-build-support/subprojects')
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'nix_api_main.cc',

13
src/libmain-c/nix_api_main.cc
View file

@ -4,6 +4,7 @@
#include "nix_api_util_internal.h"
#include "nix/main/plugin.hh"
#include "nix/main/loggers.hh"
extern "C" {
@ -17,4 +18,16 @@ nix_err nix_init_plugins(nix_c_context * context)
NIXC_CATCH_ERRS
}
nix_err nix_set_log_format(nix_c_context * context, const char * format)
{
if (context)
context->last_err_code = NIX_OK;
if (format == nullptr)
return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Log format is null");
try {
nix::setLogFormat(format);
}
NIXC_CATCH_ERRS
}
} // extern "C"

8
src/libmain-c/nix_api_main.h
View file

@ -30,6 +30,14 @@ extern "C" {
*/
nix_err nix_init_plugins(nix_c_context * context);
/**
* @brief Sets the log format
*
* @param[out] context Optional, stores error information
* @param[in] format The string name of the format.
*/
nix_err nix_set_log_format(nix_c_context * context, const char * format);
// cffi end
#ifdef __cplusplus
}

1
src/libmain/meson.build
View file

@ -53,7 +53,6 @@ config_priv_h = configure_file(
)
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'common-args.cc',

2
src/libmain/progress-bar.cc
View file

@ -183,7 +183,7 @@ public:
std::ostringstream oss;
showErrorInfo(oss, ei, loggerSettings.showTrace.get());
log(*state, ei.level, toView(oss));
log(*state, ei.level, oss.view());
}
void log(State & state, Verbosity lvl, std::string_view s)

1
src/libstore-c/meson.build
View file

@ -26,7 +26,6 @@ deps_public_maybe_subproject = [
subdir('nix-meson-build-support/subprojects')
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'nix_api_store.cc',

12
src/libstore-c/nix_api_store.cc
View file

@ -177,15 +177,17 @@ nix_err nix_store_realise(
// Check if any builds failed
for (auto & result : results) {
if (!result.success())
result.rethrow();
if (auto * failureP = result.tryGetFailure())
failureP->rethrow();
}
if (callback) {
for (const auto & result : results) {
for (const auto & [outputName, realisation] : result.builtOutputs) {
StorePath p{realisation.outPath};
callback(userdata, outputName.c_str(), &p);
if (auto * success = result.tryGetSuccess()) {
for (const auto & [outputName, realisation] : success->builtOutputs) {
StorePath p{realisation.outPath};
callback(userdata, outputName.c_str(), &p);
}
}
}
}

13
src/libstore-test-support/include/nix/store/tests/libstore.hh
View file

@ -19,14 +19,13 @@ public:
}
protected:
LibStoreTest(ref<Store> store)
: store(std::move(store))
{
}
LibStoreTest()
: store(openStore({
.variant =
StoreReference::Specified{
.scheme = "dummy",
},
.params = {},
}))
: LibStoreTest(openStore("dummy://"))
{
}

1
src/libstore-test-support/meson.build
View file

@ -29,7 +29,6 @@ rapidcheck = dependency('rapidcheck')
deps_public += rapidcheck
subdir('nix-meson-build-support/common')
subdir('nix-meson-build-support/asan-options')
sources = files(
'derived-path.cc',

0
src/libstore-tests/data/derivation/dynDerivationDeps.drv → src/libstore-tests/data/derivation/dyn-dep-derivation.drv
View file

0
src/libstore-tests/data/derivation/dynDerivationDeps.json → src/libstore-tests/data/derivation/dyn-dep-derivation.json
View file

0
src/libstore-tests/data/derivation/simple.drv → src/libstore-tests/data/derivation/simple-derivation.drv
View file

0
src/libstore-tests/data/derivation/simple.json → src/libstore-tests/data/derivation/simple-derivation.json
View file

10
src/libstore-tests/data/derived-path/multi_built_built.json Normal file
View file

@ -0,0 +1,10 @@
{
"drvPath": {
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
"output": "bar"
},
"outputs": [
"baz",
"quux"
]
}

9
src/libstore-tests/data/derived-path/multi_built_built_wildcard.json Normal file
View file

@ -0,0 +1,9 @@
{
"drvPath": {
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
"output": "bar"
},
"outputs": [
"*"
]
}

1
src/libstore-tests/data/derived-path/multi_opaque.json Normal file
View file

@ -0,0 +1 @@
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"

7
src/libstore-tests/data/derived-path/mutli_built.json Normal file
View file

@ -0,0 +1,7 @@
{
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
"outputs": [
"bar",
"baz"
]
}

4
src/libstore-tests/data/derived-path/single_built.json Normal file
View file

@ -0,0 +1,4 @@
{
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
"output": "bar"
}

7
src/libstore-tests/data/derived-path/single_built_built.json Normal file
View file

@ -0,0 +1,7 @@
{
"drvPath": {
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
"output": "bar"
},
"output": "baz"
}

1
src/libstore-tests/data/derived-path/single_opaque.json Normal file
View file

@ -0,0 +1 @@
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"

Some files were not shown because too many files have changed in this diff Show more