nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-30 06:01:00 +01:00
Code Activity

Merge branch 'master' into document-derivation-and-deriving-path

Browse source
This commit is contained in:
John Ericson 2025-01-19 11:45:55 -05:00
commit 16bd5a44ef
375 changed files with 6188 additions and 2232 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/external-api-docs/doxygen.cfg.in
View file

@ -40,6 +40,7 @@ GENERATE_LATEX = NO
INPUT = \
@src@/src/libutil-c \
@src@/src/libexpr-c \
@src@/src/libflake-c \
@src@/src/libstore-c \
@src@/src/external-api-docs/README.md

1
src/external-api-docs/package.nix
View file

@ -30,6 +30,7 @@ mkMesonDerivation (finalAttrs: {
# Source is not compiled, but still must be available for Doxygen
# to gather comments.
(cpp ../libexpr-c)
(cpp ../libflake-c)
(cpp ../libstore-c)
(cpp ../libutil-c)
];

1
src/libcmd/build-utils-meson
View file

@ -1 +0,0 @@
../../build-utils-meson

49
src/libcmd/command.cc
View file

@ -179,30 +179,34 @@ BuiltPathsCommand::BuiltPathsCommand(bool recursive)
void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
{
BuiltPaths paths;
BuiltPaths rootPaths, allPaths;
if (all) {
if (installables.size())
throw UsageError("'--all' does not expect arguments");
// XXX: Only uses opaque paths, ignores all the realisations
for (auto & p : store->queryAllValidPaths())
paths.emplace_back(BuiltPath::Opaque{p});
rootPaths.emplace_back(BuiltPath::Opaque{p});
allPaths = rootPaths;
} else {
paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
rootPaths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
allPaths = rootPaths;
if (recursive) {
// XXX: This only computes the store path closure, ignoring
// intermediate realisations
StorePathSet pathsRoots, pathsClosure;
for (auto & root : paths) {
for (auto & root : rootPaths) {
auto rootFromThis = root.outPaths();
pathsRoots.insert(rootFromThis.begin(), rootFromThis.end());
}
store->computeFSClosure(pathsRoots, pathsClosure);
for (auto & path : pathsClosure)
paths.emplace_back(BuiltPath::Opaque{path});
allPaths.emplace_back(BuiltPath::Opaque{path});
}
}
run(store, std::move(paths));
run(store, std::move(allPaths), std::move(rootPaths));
}
StorePathsCommand::StorePathsCommand(bool recursive)
@ -210,10 +214,10 @@ StorePathsCommand::StorePathsCommand(bool recursive)
{
}
void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
void StorePathsCommand::run(ref<Store> store, BuiltPaths && allPaths, BuiltPaths && rootPaths)
{
StorePathSet storePaths;
for (auto & builtPath : paths)
for (auto & builtPath : allPaths)
for (auto & p : builtPath.outPaths())
storePaths.insert(p);
@ -245,7 +249,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
{
if (!profile)
return;
auto store = getStore().dynamic_pointer_cast<LocalFSStore>();
auto store = getDstStore().dynamic_pointer_cast<LocalFSStore>();
if (!store)
throw Error("'--profile' is not supported for this Nix store");
auto profile2 = absPath(*profile);
@ -365,4 +369,31 @@ void MixEnvironment::setEnviron()
return;
}
void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store)
{
for (const auto & [_i, buildable] : enumerate(buildables)) {
auto i = _i;
std::visit(
overloaded{
[&](const BuiltPath::Opaque & bo) {
auto symlink = outLink;
if (i)
symlink += fmt("-%d", i);
store.addPermRoot(bo.path, absPath(symlink.string()));
},
[&](const BuiltPath::Built & bfd) {
for (auto & output : bfd.outputs) {
auto symlink = outLink;
if (i)
symlink += fmt("-%d", i);
if (output.first != "out")
symlink += fmt("-%s", output.first);
store.addPermRoot(output.second, absPath(symlink.string()));
}
},
},
buildable.raw());
}
}
}

26
src/libcmd/command.hh
View file

@ -18,6 +18,7 @@ extern char ** savedArgv;
class EvalState;
struct Pos;
class Store;
class LocalFSStore;
static constexpr Command::Category catHelp = -1;
static constexpr Command::Category catSecondary = 100;
@ -46,7 +47,20 @@ struct StoreCommand : virtual Command
{
StoreCommand();
void run() override;
/**
* Return the default Nix store.
*/
ref<Store> getStore();
/**
* Return the destination Nix store.
*/
virtual ref<Store> getDstStore()
{
return getStore();
}
virtual ref<Store> createStore();
/**
* Main entry point, with a `Store` provided
@ -69,7 +83,7 @@ struct CopyCommand : virtual StoreCommand
ref<Store> createStore() override;
ref<Store> getDstStore();
ref<Store> getDstStore() override;
};
/**
@ -239,7 +253,7 @@ public:
BuiltPathsCommand(bool recursive = false);
virtual void run(ref<Store> store, BuiltPaths && paths) = 0;
virtual void run(ref<Store> store, BuiltPaths && allPaths, BuiltPaths && rootPaths) = 0;
void run(ref<Store> store, Installables && installables) override;
@ -252,7 +266,7 @@ struct StorePathsCommand : public BuiltPathsCommand
virtual void run(ref<Store> store, StorePaths && storePaths) = 0;
void run(ref<Store> store, BuiltPaths && paths) override;
void run(ref<Store> store, BuiltPaths && allPaths, BuiltPaths && rootPaths) override;
};
/**
@ -354,4 +368,10 @@ std::string showVersions(const std::set<std::string> & versions);
void printClosureDiff(
ref<Store> store, const StorePath & beforePath, const StorePath & afterPath, std::string_view indent);
/**
* Create symlinks prefixed by `outLink` to the store paths in
* `buildables`.
*/
void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store);
}

6
src/libcmd/common-eval-args.cc
View file

@ -29,13 +29,13 @@ EvalSettings evalSettings {
{
{
"flake",
[](ref<Store> store, std::string_view rest) {
[](EvalState & state, std::string_view rest) {
experimentalFeatureSettings.require(Xp::Flakes);
// FIXME `parseFlakeRef` should take a `std::string_view`.
auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
debug("fetching flake search path element '%s''", rest);
auto storePath = flakeRef.resolve(store).fetchTree(store).first;
return store->toRealPath(storePath);
auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
return state.rootPath(state.store->toRealPath(storePath));
},
},
},

12
src/libcmd/installables.cc
View file

@ -450,7 +450,7 @@ ref<eval_cache::EvalCache> openEvalCache(
std::shared_ptr<flake::LockedFlake> lockedFlake)
{
auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval
? lockedFlake->getFingerprint(state.store)
? lockedFlake->getFingerprint(state.store, state.fetchSettings)
: std::nullopt;
auto rootLoader = [&state, lockedFlake]()
{
@ -858,7 +858,7 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
applyDefaultInstallables(rawInstallables);
std::vector<FlakeRef> res;
res.reserve(rawInstallables.size());
for (auto i : rawInstallables)
for (const auto & i : rawInstallables)
res.push_back(parseFlakeRefWithFragment(
fetchSettings,
expandTilde(i),
@ -918,4 +918,12 @@ void BuiltPathsCommand::applyDefaultInstallables(std::vector<std::string> & rawI
rawInstallables.push_back(".");
}
BuiltPaths toBuiltPaths(const std::vector<BuiltPathWithResult> & builtPathsWithResult)
{
BuiltPaths res;
for (auto & i : builtPathsWithResult)
res.push_back(i.path);
return res;
}
}

2
src/libcmd/installables.hh
View file

@ -86,6 +86,8 @@ struct BuiltPathWithResult
std::optional<BuildResult> result;
};
BuiltPaths toBuiltPaths(const std::vector<BuiltPathWithResult> & builtPathsWithResult);
/**
* Shorthand, for less typing and helping us keep the choice of
* collection in sync.

20
src/libcmd/markdown.cc
View file

@ -16,13 +16,25 @@ static std::string doRenderMarkdownToTerminal(std::string_view markdown)
{
int windowWidth = getWindowSize().second;
struct lowdown_opts opts
{
.type = LOWDOWN_TERM,
.maxdepth = 20,
#if HAVE_LOWDOWN_1_4
struct lowdown_opts_term opts_term {
.cols = (size_t) std::max(windowWidth - 5, 60),
.hmargin = 0,
.vmargin = 0,
};
#endif
struct lowdown_opts opts
{
.type = LOWDOWN_TERM,
#if HAVE_LOWDOWN_1_4
.term = opts_term,
#endif
.maxdepth = 20,
#if !HAVE_LOWDOWN_1_4
.cols = (size_t) std::max(windowWidth - 5, 60),
.hmargin = 0,
.vmargin = 0,
#endif
.feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
.oflags = LOWDOWN_TERM_NOLINK,
};

14
src/libcmd/meson.build
View file

@ -4,8 +4,6 @@ project('nix-cmd', 'cpp',
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
@ -14,7 +12,7 @@ project('nix-cmd', 'cpp',
cxx = meson.get_compiler('cpp')
subdir('build-utils-meson/deps-lists')
subdir('nix-meson-build-support/deps-lists')
configdata = configuration_data()
@ -28,9 +26,7 @@ deps_public_maybe_subproject = [
dependency('nix-flake'),
dependency('nix-main'),
]
subdir('build-utils-meson/subprojects')
subdir('build-utils-meson/threads')
subdir('nix-meson-build-support/subprojects')
nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
deps_public += nlohmann_json
@ -38,6 +34,8 @@ deps_public += nlohmann_json
lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown'))
deps_private += lowdown
configdata.set('HAVE_LOWDOWN', lowdown.found().to_int())
# The API changed slightly around terminal initialization.
configdata.set('HAVE_LOWDOWN_1_4', lowdown.version().version_compare('>= 1.4.0').to_int())
readline_flavor = get_option('readline-flavor')
if readline_flavor == 'editline'
@ -72,7 +70,7 @@ add_project_arguments(
language : 'cpp',
)
subdir('build-utils-meson/diagnostics')
subdir('nix-meson-build-support/common')
sources = files(
'built-path.cc',
@ -127,4 +125,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true)
libraries_private = []
subdir('build-utils-meson/export')
subdir('nix-meson-build-support/export')

1
src/libcmd/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

8
src/libcmd/package.nix
View file

@ -39,8 +39,8 @@ mkMesonLibrary (finalAttrs: {
workDir = ./.;
fileset = fileset.unions [
../../build-utils-meson
./build-utils-meson
../../nix-meson-build-support
./nix-meson-build-support
../../.version
./.version
./meson.build
@ -76,10 +76,6 @@ mkMesonLibrary (finalAttrs: {
(lib.mesonOption "readline-flavor" readlineFlavor)
];
env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
meta = {
platforms = lib.platforms.unix ++ lib.platforms.windows;
};

1
src/libexpr-c/build-utils-meson
View file

@ -1 +0,0 @@
../../build-utils-meson

16
src/libexpr-c/meson.build
View file

@ -4,8 +4,6 @@ project('nix-expr-c', 'cpp',
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
@ -14,7 +12,7 @@ project('nix-expr-c', 'cpp',
cxx = meson.get_compiler('cpp')
subdir('build-utils-meson/deps-lists')
subdir('nix-meson-build-support/deps-lists')
configdata = configuration_data()
@ -27,9 +25,7 @@ deps_public_maybe_subproject = [
dependency('nix-util-c'),
dependency('nix-store-c'),
]
subdir('build-utils-meson/subprojects')
subdir('build-utils-meson/threads')
subdir('nix-meson-build-support/subprojects')
# TODO rename, because it will conflict with downstream projects
configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
@ -55,7 +51,7 @@ add_project_arguments(
language : 'cpp',
)
subdir('build-utils-meson/diagnostics')
subdir('nix-meson-build-support/common')
sources = files(
'nix_api_expr.cc',
@ -74,8 +70,8 @@ headers = [config_h] + files(
# TODO move this header to libexpr, maybe don't use it in tests?
headers += files('nix_api_expr_internal.h')
subdir('build-utils-meson/export-all-symbols')
subdir('build-utils-meson/windows-version')
subdir('nix-meson-build-support/export-all-symbols')
subdir('nix-meson-build-support/windows-version')
this_library = library(
'nixexprc',
@ -91,4 +87,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true)
libraries_private = []
subdir('build-utils-meson/export')
subdir('nix-meson-build-support/export')

1
src/libexpr-c/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

116
src/libexpr-c/nix_api_expr.cc
View file

@ -6,6 +6,7 @@
#include "eval-gc.hh"
#include "globals.hh"
#include "eval-settings.hh"
#include "ref.hh"
#include "nix_api_expr.h"
#include "nix_api_expr_internal.h"
@ -18,6 +19,29 @@
# include <mutex>
#endif
/**
* @brief Allocate and initialize using self-reference
*
* This allows a brace initializer to reference the object being constructed.
*
* @warning Use with care, as the pointer points to an object that is not fully constructed yet.
*
* @tparam T Type to allocate
* @tparam F A function type for `init`, taking a T* and returning the initializer for T
* @param init Function that takes a T* and returns the initializer for T
* @return Pointer to allocated and initialized object
*/
template <typename T, typename F>
static T * unsafe_new_with_self(F && init)
{
// Allocate
void * p = ::operator new(
sizeof(T),
static_cast<std::align_val_t>(alignof(T)));
// Initialize with placement new
return new (p) T(init(static_cast<T *>(p)));
}
nix_err nix_libexpr_init(nix_c_context * context)
{
if (context)
@ -67,7 +91,7 @@ nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_val
if (context)
context->last_err_code = NIX_OK;
try {
state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos);
state->state.callFunction(fn->value, {(nix::Value * *) args, nargs}, value->value, nix::noPos);
state->state.forceValue(value->value, nix::noPos);
}
NIXC_CATCH_ERRS
@ -93,7 +117,42 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val
NIXC_CATCH_ERRS
}
EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store)
nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store)
{
if (context)
context->last_err_code = NIX_OK;
try {
return unsafe_new_with_self<nix_eval_state_builder>([&](auto * self) {
return nix_eval_state_builder{
.store = nix::ref<nix::Store>(store->ptr),
.settings = nix::EvalSettings{/* &bool */ self->readOnlyMode},
.fetchSettings = nix::fetchers::Settings{},
.readOnlyMode = true,
};
});
}
NIXC_CATCH_ERRS_NULL
}
void nix_eval_state_builder_free(nix_eval_state_builder * builder)
{
delete builder;
}
nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder)
{
if (context)
context->last_err_code = NIX_OK;
try {
// TODO: load in one go?
builder->settings.readOnlyMode = nix::settings.readOnlyMode;
loadConfFile(builder->settings);
loadConfFile(builder->fetchSettings);
}
NIXC_CATCH_ERRS
}
nix_err nix_eval_state_builder_set_lookup_path(nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath_c)
{
if (context)
context->last_err_code = NIX_OK;
@ -102,28 +161,47 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
if (lookupPath_c != nullptr)
for (size_t i = 0; lookupPath_c[i] != nullptr; i++)
lookupPath.push_back(lookupPath_c[i]);
builder->lookupPath = nix::LookupPath::parse(lookupPath);
}
NIXC_CATCH_ERRS
}
void * p = ::operator new(
sizeof(EvalState),
static_cast<std::align_val_t>(alignof(EvalState)));
auto * p2 = static_cast<EvalState *>(p);
new (p) EvalState {
.fetchSettings = nix::fetchers::Settings{},
.settings = nix::EvalSettings{
nix::settings.readOnlyMode,
},
.state = nix::EvalState(
nix::LookupPath::parse(lookupPath),
store->ptr,
p2->fetchSettings,
p2->settings),
};
loadConfFile(p2->settings);
return p2;
EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder)
{
if (context)
context->last_err_code = NIX_OK;
try {
return unsafe_new_with_self<EvalState>([&](auto * self) {
return EvalState{
.fetchSettings = std::move(builder->fetchSettings),
.settings = std::move(builder->settings),
.state = nix::EvalState(
builder->lookupPath,
builder->store,
self->fetchSettings,
self->settings),
};
});
}
NIXC_CATCH_ERRS_NULL
}
EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store)
{
auto builder = nix_eval_state_builder_new(context, store);
if (builder == nullptr)
return nullptr;
if (nix_eval_state_builder_load(context, builder) != NIX_OK)
return nullptr;
if (nix_eval_state_builder_set_lookup_path(context, builder, lookupPath_c)
!= NIX_OK)
return nullptr;
return nix_eval_state_build(context, builder);
}
void nix_state_free(EvalState * state)
{
delete state;

65
src/libexpr-c/nix_api_expr.h
View file

@ -30,6 +30,11 @@ extern "C" {
// cffi start
// Type definitions
/**
* @brief Builder for EvalState
*/
typedef struct nix_eval_state_builder nix_eval_state_builder;
/**
* @brief Represents a state of the Nix language evaluator.
*
@ -174,12 +179,70 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value *
nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value);
/**
* @brief Create a new Nix language evaluator state.
* @brief Create a new nix_eval_state_builder
*
* The settings are initialized to their default value.
* Values can be sourced elsewhere with nix_eval_state_builder_load.
*
* @param[out] context Optional, stores error information
* @param[in] store The Nix store to use.
* @return A new nix_eval_state_builder or NULL on failure.
*/
nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store);
/**
* @brief Read settings from the ambient environment
*
* Settings are sourced from environment variables and configuration files,
* as documented in the Nix manual.
*
* @param[out] context Optional, stores error information
* @param[out] builder The builder to modify.
* @return NIX_OK if successful, an error code otherwise.
*/
nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder);
/**
* @brief Set the lookup path for `<...>` expressions
*
* @param[in] context Optional, stores error information
* @param[in] builder The builder to modify.
* @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
*/
nix_err nix_eval_state_builder_set_lookup_path(
nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath);
/**
* @brief Create a new Nix language evaluator state
*
* Remember to nix_eval_state_builder_free after building the state.
*
* @param[out] context Optional, stores error information
* @param[in] builder The builder to use and free
* @return A new Nix state or NULL on failure.
* @see nix_eval_state_builder_new, nix_eval_state_builder_free
*/
EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder);
/**
* @brief Free a nix_eval_state_builder
*
* Does not fail.
*
* @param[in] builder The builder to free.
*/
void nix_eval_state_builder_free(nix_eval_state_builder * builder);
/**
* @brief Create a new Nix language evaluator state
*
* For more control, use nix_eval_state_builder
*
* @param[out] context Optional, stores error information
* @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
* @param[in] store The Nix store to use.
* @return A new Nix state or NULL on failure.
* @see nix_state_builder_new
*/
EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store);

11
src/libexpr-c/nix_api_expr_internal.h
View file

@ -6,6 +6,17 @@
#include "eval-settings.hh"
#include "attr-set.hh"
#include "nix_api_value.h"
#include "search-path.hh"
struct nix_eval_state_builder
{
nix::ref<nix::Store> store;
nix::EvalSettings settings;
nix::fetchers::Settings fetchSettings;
nix::LookupPath lookupPath;
// TODO: make an EvalSettings setting own this instead?
bool readOnlyMode;
};
struct EvalState
{

2
src/libexpr-c/nix_api_value.h
View file

@ -213,7 +213,7 @@ nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_
/** @brief Get path as string
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return string
* @return string, if the type is NIX_TYPE_PATH
* @return NULL in case of error.
*/
const char * nix_get_path_string(nix_c_context * context, const nix_value * value);

9
src/libexpr-c/package.nix
View file

@ -1,5 +1,4 @@
{ lib
, stdenv
, mkMesonLibrary
, nix-store-c
@ -20,8 +19,8 @@ mkMesonLibrary (finalAttrs: {
workDir = ./.;
fileset = fileset.unions [
../../build-utils-meson
./build-utils-meson
../../nix-meson-build-support
./nix-meson-build-support
../../.version
./.version
./meson.build
@ -47,10 +46,6 @@ mkMesonLibrary (finalAttrs: {
mesonFlags = [
];
env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
meta = {
platforms = lib.platforms.unix ++ lib.platforms.windows;
};

1
src/libexpr-test-support/build-utils-meson
View file

@ -1 +0,0 @@
../../build-utils-meson

17
src/libexpr-test-support/meson.build
View file

@ -4,8 +4,6 @@ project('nix-expr-test-support', 'cpp',
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
@ -14,7 +12,7 @@ project('nix-expr-test-support', 'cpp',
cxx = meson.get_compiler('cpp')
subdir('build-utils-meson/deps-lists')
subdir('nix-meson-build-support/deps-lists')
deps_private_maybe_subproject = [
]
@ -24,10 +22,9 @@ deps_public_maybe_subproject = [
dependency('nix-store'),
dependency('nix-store-test-support'),
dependency('nix-expr'),
dependency('nix-expr-c'),
]
subdir('build-utils-meson/subprojects')
subdir('build-utils-meson/threads')
subdir('nix-meson-build-support/subprojects')
rapidcheck = dependency('rapidcheck')
deps_public += rapidcheck
@ -41,7 +38,7 @@ add_project_arguments(
language : 'cpp',
)
subdir('build-utils-meson/diagnostics')
subdir('nix-meson-build-support/common')
sources = files(
'tests/value/context.cc',
@ -55,8 +52,8 @@ headers = files(
'tests/value/context.hh',
)
subdir('build-utils-meson/export-all-symbols')
subdir('build-utils-meson/windows-version')
subdir('nix-meson-build-support/export-all-symbols')
subdir('nix-meson-build-support/windows-version')
this_library = library(
'nix-expr-test-support',
@ -74,4 +71,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true)
libraries_private = []
subdir('build-utils-meson/export')
subdir('nix-meson-build-support/export')

1
src/libexpr-test-support/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

11
src/libexpr-test-support/package.nix
View file

@ -1,9 +1,9 @@
{ lib
, stdenv
, mkMesonLibrary
, nix-store-test-support
, nix-expr
, nix-expr-c
, rapidcheck
@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: {
workDir = ./.;
fileset = fileset.unions [
../../build-utils-meson
./build-utils-meson
../../nix-meson-build-support
./nix-meson-build-support
../../.version
./.version
./meson.build
@ -35,6 +35,7 @@ mkMesonLibrary (finalAttrs: {
propagatedBuildInputs = [
nix-store-test-support
nix-expr
nix-expr-c
rapidcheck
];
@ -49,10 +50,6 @@ mkMesonLibrary (finalAttrs: {
mesonFlags = [
];
env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
meta = {
platforms = lib.platforms.unix ++ lib.platforms.windows;
};

6
src/libexpr-test-support/tests/libexpr.hh
View file

@ -40,6 +40,12 @@ namespace nix {
return v;
}
Value * maybeThunk(std::string input, bool forceValue = true) {
Expr * e = state.parseExprFromString(input, state.rootPath(CanonPath::root));
assert(e);
return e->maybeThunk(state, state.baseEnv);
}
Symbol createSymbol(const char * value) {
return state.symbols.create(value);
}

1
src/libexpr-tests/build-utils-meson
View file

@ -1 +0,0 @@
../../build-utils-meson

14
src/libexpr-tests/error_traces.cc
View file

@ -691,15 +691,15 @@ namespace nix {
ASSERT_TRACE2("elemAt \"foo\" (-1)",
TypeError,
HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)),
HintFmt("while evaluating the first argument passed to builtins.elemAt"));
HintFmt("while evaluating the first argument passed to 'builtins.elemAt'"));
ASSERT_TRACE1("elemAt [] (-1)",
Error,
HintFmt("list index %d is out of bounds", -1));
HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0));
ASSERT_TRACE1("elemAt [\"foo\"] 3",
Error,
HintFmt("list index %d is out of bounds", 3));
HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1));
}
@ -708,11 +708,11 @@ namespace nix {
ASSERT_TRACE2("head 1",
TypeError,
HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)),
HintFmt("while evaluating the first argument passed to builtins.elemAt"));
HintFmt("while evaluating the first argument passed to 'builtins.head'"));
ASSERT_TRACE1("head []",
Error,
HintFmt("list index %d is out of bounds", 0));
HintFmt("'builtins.head' called on an empty list"));
}
@ -721,11 +721,11 @@ namespace nix {
ASSERT_TRACE2("tail 1",
TypeError,
HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)),
HintFmt("while evaluating the first argument passed to builtins.tail"));
HintFmt("while evaluating the first argument passed to 'builtins.tail'"));
ASSERT_TRACE1("tail []",
Error,
HintFmt("'tail' called on an empty list"));
HintFmt("'builtins.tail' called on an empty list"));
}

25
src/libexpr-tests/eval.cc
View file

@ -138,4 +138,27 @@ TEST(nix_isAllowedURI, non_scheme_colon) {
ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed));
}
} // namespace nix
class EvalStateTest : public LibExprTest {};
TEST_F(EvalStateTest, getBuiltins_ok) {
auto evaled = maybeThunk("builtins");
auto & builtins = state.getBuiltins();
ASSERT_TRUE(builtins.type() == nAttrs);
ASSERT_EQ(evaled, &builtins);
}
TEST_F(EvalStateTest, getBuiltin_ok) {
auto & builtin = state.getBuiltin("toString");
ASSERT_TRUE(builtin.type() == nFunction);
// FIXME
// auto evaled = maybeThunk("builtins.toString");
// ASSERT_EQ(evaled, &builtin);
auto & builtin2 = state.getBuiltin("true");
ASSERT_EQ(state.forceBool(builtin2, noPos, "in unit test"), true);
}
TEST_F(EvalStateTest, getBuiltin_fail) {
ASSERT_THROW(state.getBuiltin("nonexistent"), EvalError);
}
} // namespace nix

14
src/libexpr-tests/meson.build
View file

@ -4,8 +4,6 @@ project('nix-expr-tests', 'cpp',
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
@ -14,7 +12,7 @@ project('nix-expr-tests', 'cpp',
cxx = meson.get_compiler('cpp')
subdir('build-utils-meson/deps-lists')
subdir('nix-meson-build-support/deps-lists')
deps_private_maybe_subproject = [
dependency('nix-expr'),
@ -23,12 +21,10 @@ deps_private_maybe_subproject = [
]
deps_public_maybe_subproject = [
]
subdir('build-utils-meson/subprojects')
subdir('nix-meson-build-support/subprojects')
subdir('build-utils-meson/threads')
subdir('build-utils-meson/export-all-symbols')
subdir('build-utils-meson/windows-version')
subdir('nix-meson-build-support/export-all-symbols')
subdir('nix-meson-build-support/windows-version')
rapidcheck = dependency('rapidcheck')
deps_private += rapidcheck
@ -51,7 +47,7 @@ add_project_arguments(
language : 'cpp',
)
subdir('build-utils-meson/diagnostics')
subdir('nix-meson-build-support/common')
sources = files(
'derived-path.cc',

1
src/libexpr-tests/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

37
src/libexpr-tests/nix_api_expr.cc
View file

@ -7,12 +7,49 @@
#include "tests/nix_api_expr.hh"
#include "tests/string_callback.hh"
#include "file-system.hh"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
namespace nixC {
TEST_F(nix_api_store_test, nix_eval_state_lookup_path)
{
auto tmpDir = nix::createTempDir();
auto delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true);
auto nixpkgs = tmpDir + "/pkgs";
auto nixos = tmpDir + "/cfg";
std::filesystem::create_directories(nixpkgs);
std::filesystem::create_directories(nixos);
std::string nixpkgsEntry = "nixpkgs=" + nixpkgs;
std::string nixosEntry = "nixos-config=" + nixos;
const char * lookupPath[] = {nixpkgsEntry.c_str(), nixosEntry.c_str(), nullptr};
auto builder = nix_eval_state_builder_new(ctx, store);
assert_ctx_ok();
ASSERT_EQ(NIX_OK, nix_eval_state_builder_set_lookup_path(ctx, builder, lookupPath));
assert_ctx_ok();
auto state = nix_eval_state_build(ctx, builder);
assert_ctx_ok();
nix_eval_state_builder_free(builder);
Value * value = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(ctx, state, "builtins.seq <nixos-config> <nixpkgs>", ".", value);
assert_ctx_ok();
ASSERT_EQ(nix_get_type(ctx, value), NIX_TYPE_PATH);
assert_ctx_ok();
auto pathStr = nix_get_path_string(ctx, value);
assert_ctx_ok();
ASSERT_EQ(0, strcmp(pathStr, nixpkgs.c_str()));
}
TEST_F(nix_api_expr_test, nix_expr_eval_from_string)
{
nix_expr_eval_from_string(nullptr, state, "builtins.nixVersion", ".", value);

8
src/libexpr-tests/package.nix
View file

@ -27,8 +27,8 @@ mkMesonExecutable (finalAttrs: {
workDir = ./.;
fileset = fileset.unions [
../../build-utils-meson
./build-utils-meson
../../nix-meson-build-support
./nix-meson-build-support
../../.version
./.version
./meson.build
@ -56,10 +56,6 @@ mkMesonExecutable (finalAttrs: {
mesonFlags = [
];
env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
passthru = {
tests = {
run = runCommand "${finalAttrs.pname}-run" {

51
src/libexpr-tests/trivial.cc
View file

@ -177,6 +177,57 @@ namespace nix {
)
);
// The following macros ultimately define 48 tests (16 variations on three
// templates). Each template tests an expression that can be written in 2^4
// different ways, by making four choices about whether to write a particular
// attribute path segment as `x.y = ...;` (collapsed) or `x = { y = ...; };`
// (expanded).
//
// The nestedAttrsetMergeXXXX tests check that the expression
// `{ a.b.c = 1; a.b.d = 2; }` has the same value regardless of how it is
// expanded. (That exact expression is exercised in test
// nestedAttrsetMerge0000, because it is fully collapsed. The test
// nestedAttrsetMerge1001 would instead examine
// `{ a = { b.c = 1; }; a.b = { d = 2; }; }`.)
//
// The nestedAttrsetMergeDupXXXX tests check that the expression
// `{ a.b.c = 1; a.b.c = 2; }` throws a duplicate attribute error, again
// regardless of how it is expanded.
//
// The nestedAttrsetMergeLetXXXX tests check that the expression
// `let a.b.c = 1; a.b.d = 2; in a` has the same value regardless of how it is
// expanded.
#define X_EXPAND_IF0(k, v) k "." v
#define X_EXPAND_IF1(k, v) k " = { " v " };"
#define X4(w, x, y, z) \
TEST_F(TrivialExpressionTest, nestedAttrsetMerge##w##x##y##z) { \
auto v = eval("{ a.b = { c = 1; d = 2; }; } == { " \
X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \
X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " }"); \
ASSERT_THAT(v, IsTrue()); \
}; \
TEST_F(TrivialExpressionTest, nestedAttrsetMergeDup##w##x##y##z) { \
ASSERT_THROW(eval("{ " \
X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \
X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "c = 2;")) " }"), Error); \
}; \
TEST_F(TrivialExpressionTest, nestedAttrsetMergeLet##w##x##y##z) { \
auto v = eval("{ b = { c = 1; d = 2; }; } == (let " \
X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \
X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " in a)"); \
ASSERT_THAT(v, IsTrue()); \
};
#define X3(...) X4(__VA_ARGS__, 0) X4(__VA_ARGS__, 1)
#define X2(...) X3(__VA_ARGS__, 0) X3(__VA_ARGS__, 1)
#define X1(...) X2(__VA_ARGS__, 0) X2(__VA_ARGS__, 1)
X1(0) X1(1)
#undef X_EXPAND_IF0
#undef X_EXPAND_IF1
#undef X1
#undef X2
#undef X3
#undef X4
TEST_F(TrivialExpressionTest, functor) {
auto v = eval("{ __functor = self: arg: self.v + arg; v = 10; } 5");
ASSERT_THAT(v, IsIntEq(15));

1
src/libexpr/attr-path.cc
View file

@ -129,7 +129,6 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
try {
auto colon = fn.rfind(':');
if (colon == std::string::npos) fail();
std::string filename(fn, 0, colon);
auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
} catch (std::invalid_argument & e) {

1
src/libexpr/build-utils-meson
View file

@ -1 +0,0 @@
../../build-utils-meson

7
src/libexpr/call-flake.nix
View file

@ -41,10 +41,17 @@ let
(key: node:
let
parentNode = allNodes.${getInputByPath lockFile.root node.parent};
sourceInfo =
if overrides ? ${key}
then
overrides.${key}.sourceInfo
else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/"
then
parentNode.sourceInfo // {
outPath = parentNode.outPath + ("/" + node.locked.path);
}
else
# FIXME: remove obsolete node.info.
# Note: lock file entries are always final.

6
src/libexpr/eval-inline.hh
View file

@ -87,11 +87,15 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
{
if (v.isThunk()) {
Env * env = v.payload.thunk.env;
assert(env || v.isBlackhole());
Expr * expr = v.payload.thunk.expr;
try {
v.mkBlackhole();
//checkInterrupt();
expr->eval(*this, *env, v);
if (env) [[likely]]
expr->eval(*this, *env, v);
else
ExprBlackHole::throwInfiniteRecursionError(*this, v);
} catch (...) {
v.mkThunk(env, expr);
tryFixupBlackHolePos(v, pos);

8
src/libexpr/eval-settings.hh
View file

@ -3,10 +3,11 @@
#include "config.hh"
#include "ref.hh"
#include "source-path.hh"
namespace nix {
class Store;
class EvalState;
struct EvalSettings : Config
{
@ -18,11 +19,8 @@ struct EvalSettings : Config
*
* The return value is (a) whether the entry was valid, and, if so,
* what does it map to.
*
* @todo Return (`std::optional` of) `SourceAccssor` or something
* more structured instead of mere `std::string`?
*/
using LookupPathHook = std::optional<std::string>(ref<Store> store, std::string_view);
using LookupPathHook = std::optional<SourcePath>(EvalState & state, std::string_view);
/**
* Map from "scheme" to a `LookupPathHook`.

108
src/libexpr/eval.cc
View file

@ -347,6 +347,16 @@ void EvalState::allowPath(const StorePath & storePath)
rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath)));
}
void EvalState::allowClosure(const StorePath & storePath)
{
if (!rootFS.dynamic_pointer_cast<AllowListSourceAccessor>()) return;
StorePathSet closure;
store->computeFSClosure(storePath, closure);
for (auto & p : closure)
allowPath(p);
}
void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
{
allowPath(storePath);
@ -396,7 +406,7 @@ void EvalState::checkURI(const std::string & uri)
/* If the URI is a path, then check it against allowedPaths as
well. */
if (hasPrefix(uri, "/")) {
if (isAbsolute(uri)) {
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
rootFS2->checkAccess(CanonPath(uri));
return;
@ -448,7 +458,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info)
/* Install value the base environment. */
staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(name2), v));
getBuiltins().payload.attrs->push_back(Attr(symbols.create(name2), v));
}
}
@ -516,16 +526,26 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
else {
staticBaseEnv->vars.emplace_back(envName, baseEnvDispl);
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(primOp.name), v));
getBuiltins().payload.attrs->push_back(Attr(symbols.create(primOp.name), v));
}
return v;
}
Value & EvalState::getBuiltins()
{
return *baseEnv.values[0];
}
Value & EvalState::getBuiltin(const std::string & name)
{
return *baseEnv.values[0]->attrs()->find(symbols.create(name))->value;
auto it = getBuiltins().attrs()->get(symbols.create(name));
if (it)
return *it->value;
else
error<EvalError>("builtin '%1%' not found", name).debugThrow();
}
@ -588,14 +608,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
if (isFunctor(v)) {
try {
Value & functor = *v.attrs()->find(sFunctor)->value;
Value * vp = &v;
Value * vp[] = {&v};
Value partiallyApplied;
// The first paramater is not user-provided, and may be
// handled by code that is opaque to the user, like lib.const = x: y: y;
// So preferably we show docs that are relevant to the
// "partially applied" function returned by e.g. `const`.
// We apply the first argument:
callFunction(functor, 1, &vp, partiallyApplied, noPos);
callFunction(functor, vp, partiallyApplied, noPos);
auto _level = addCallDepth(noPos);
return getDoc(partiallyApplied);
}
@ -1460,7 +1480,7 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
v.mkLambda(&env, this);
}
void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes, const PosIdx pos)
{
auto _level = addCallDepth(pos);
@ -1475,16 +1495,16 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
auto makeAppChain = [&]()
{
vRes = vCur;
for (size_t i = 0; i < nrArgs; ++i) {
for (auto arg : args) {
auto fun2 = allocValue();
*fun2 = vRes;
vRes.mkPrimOpApp(fun2, args[i]);
vRes.mkPrimOpApp(fun2, arg);
}
};
const Attr * functor;
while (nrArgs > 0) {
while (args.size() > 0) {
if (vCur.isLambda()) {
@ -1587,15 +1607,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
throw;
}
nrArgs--;
args += 1;
args = args.subspan(1);
}
else if (vCur.isPrimOp()) {
size_t argsLeft = vCur.primOp()->arity;
if (nrArgs < argsLeft) {
if (args.size() < argsLeft) {
/* We don't have enough arguments, so create a tPrimOpApp chain. */
makeAppChain();
return;
@ -1607,15 +1626,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
if (countCalls) primOpCalls[fn->name]++;
try {
fn->fun(*this, vCur.determinePos(noPos), args, vCur);
fn->fun(*this, vCur.determinePos(noPos), args.data(), vCur);
} catch (Error & e) {
if (fn->addTrace)
addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
throw;
}
nrArgs -= argsLeft;
args += argsLeft;
args = args.subspan(argsLeft);
}
}
@ -1631,7 +1649,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
auto arity = primOp->primOp()->arity;
auto argsLeft = arity - argsDone;
if (nrArgs < argsLeft) {
if (args.size() < argsLeft) {
/* We still don't have enough arguments, so extend the tPrimOpApp chain. */
makeAppChain();
return;
@ -1663,8 +1681,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
throw;
}
nrArgs -= argsLeft;
args += argsLeft;
args = args.subspan(argsLeft);
}
}
@ -1675,13 +1692,12 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
Value * args2[] = {allocValue(), args[0]};
*args2[0] = vCur;
try {
callFunction(*functor->value, 2, args2, vCur, functor->pos);
callFunction(*functor->value, args2, vCur, functor->pos);
} catch (Error & e) {
e.addTrace(positions[pos], "while calling a functor (an attribute set with a '__functor' attribute)");
throw;
}
nrArgs--;
args++;
args = args.subspan(1);
}
else
@ -1724,7 +1740,7 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
for (size_t i = 0; i < args.size(); ++i)
vArgs[i] = args[i]->maybeThunk(state, env);
state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
state.callFunction(vFun, vArgs, v, pos);
}
@ -2046,9 +2062,12 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v)
state.mkPos(v, pos);
}
void ExprBlackHole::eval(EvalState & state, Env & env, Value & v)
void ExprBlackHole::eval(EvalState & state, [[maybe_unused]] Env & env, Value & v)
{
throwInfiniteRecursionError(state, v);
}
[[gnu::noinline]] [[noreturn]] void ExprBlackHole::throwInfiniteRecursionError(EvalState & state, Value &v) {
state.error<InfiniteRecursionError>("infinite recursion encountered")
.atPos(v.determinePos(noPos))
.debugThrow();
@ -3029,8 +3048,8 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
if (!rOpt) continue;
auto r = *rOpt;
Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
auto res = (r / CanonPath(suffix)).resolveSymlinks();
if (res.pathExists()) return res;
}
if (hasPrefix(path, "nix/"))
@ -3045,13 +3064,13 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
}
std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
{
auto & value = value0.s;
auto i = lookupPathResolved.find(value);
if (i != lookupPathResolved.end()) return i->second;
auto finish = [&](std::string res) {
auto finish = [&](SourcePath res) {
debug("resolved search path element '%s' to '%s'", value, res);
lookupPathResolved.emplace(value, res);
return res;
@ -3064,7 +3083,7 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
fetchSettings,
EvalSettings::resolvePseudoUrl(value));
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
return finish(store->toRealPath(storePath));
return finish(rootPath(store->toRealPath(storePath)));
} catch (Error & e) {
logWarning({
.msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
@ -3076,29 +3095,26 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
auto scheme = value.substr(0, colPos);
auto rest = value.substr(colPos + 1);
if (auto * hook = get(settings.lookupPathHooks, scheme)) {
auto res = (*hook)(store, rest);
auto res = (*hook)(*this, rest);
if (res)
return finish(std::move(*res));
}
}
{
auto path = absPath(value);
auto path = rootPath(value);
/* Allow access to paths in the search path. */
if (initAccessControl) {
allowPath(path);
if (store->isInStore(path)) {
allowPath(path.path.abs());
if (store->isInStore(path.path.abs())) {
try {
StorePathSet closure;
store->computeFSClosure(store->toStorePath(path).first, closure);
for (auto & p : closure)
allowPath(p);
allowClosure(store->toStorePath(path.path.abs()).first);
} catch (InvalidPath &) { }
}
}
if (pathExists(path))
if (path.pathExists())
return finish(std::move(path));
else {
logWarning({
@ -3109,7 +3125,6 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
debug("failed to resolve search path element '%s'", value);
return std::nullopt;
}
@ -3170,5 +3185,18 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
return v.print(str);
}
void forceNoNullByte(std::string_view s, std::function<Pos()> pos)
{
if (s.find('\0') != s.npos) {
using namespace std::string_view_literals;
auto str = replaceStrings(std::string(s), "\0"sv, "␀"sv);
Error error("input string '%s' cannot be represented as Nix string because it contains null bytes", str);
if (pos) {
error.atPos(pos());
}
throw error;
}
}
}

28
src/libexpr/eval.hh
View file

@ -347,7 +347,7 @@ private:
LookupPath lookupPath;
std::map<std::string, std::optional<std::string>> lookupPathResolved;
std::map<std::string, std::optional<SourcePath>> lookupPathResolved;
/**
* Cache used by prim_match().
@ -400,6 +400,11 @@ public:
*/
void allowPath(const StorePath & storePath);
/**
* Allow access to the closure of a store path.
*/
void allowClosure(const StorePath & storePath);
/**
* Allow access to a store path and return it as a string.
*/
@ -452,9 +457,9 @@ public:
*
* If the specified search path element is a URI, download it.
*
* If it is not found, return `std::nullopt`
* If it is not found, return `std::nullopt`.
*/
std::optional<std::string> resolveLookupPathPath(
std::optional<SourcePath> resolveLookupPathPath(
const LookupPath::Path & elem,
bool initAccessControl = false);
@ -623,8 +628,19 @@ private:
public:
/**
* Retrieve a specific builtin, equivalent to evaluating `builtins.${name}`.
* @param name The attribute name of the builtin to retrieve.
* @throws EvalError if the builtin does not exist.
*/
Value & getBuiltin(const std::string & name);
/**
* Retrieve the `builtins` attrset, equivalent to evaluating the reference `builtins`.
* Always returns an attribute set value.
*/
Value & getBuiltins();
struct Doc
{
Pos pos;
@ -690,13 +706,12 @@ public:
bool isFunctor(Value & fun);
// FIXME: use std::span
void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos);
void callFunction(Value & fun, std::span<Value *> args, Value & vRes, const PosIdx pos);
void callFunction(Value & fun, Value & arg, Value & vRes, const PosIdx pos)
{
Value * args[] = {&arg};
callFunction(fun, 1, args, vRes, pos);
callFunction(fun, args, vRes, pos);
}
/**
@ -809,7 +824,6 @@ public:
bool callPathFilter(
Value * filterFun,
const SourcePath & path,
std::string_view pathArg,
PosIdx pos);
DocComment getDocCommentForPos(PosIdx pos);

2
src/libexpr/json-to-value.cc
View file

@ -50,6 +50,7 @@ class JSONSax : nlohmann::json_sax<json> {
public:
void key(string_t & name, EvalState & state)
{
forceNoNullByte(name);
attrs.insert_or_assign(state.symbols.create(name), &value(state));
}
};
@ -122,6 +123,7 @@ public:
bool string(string_t & val) override
{
forceNoNullByte(val);
rs->value(state).mkString(val);
rs->add();
return true;

8
src/libexpr/lexer-helpers.hh
View file

@ -1,5 +1,13 @@
#pragma once
#include <cstddef>
// inluding the generated headers twice leads to errors
#ifndef BISON_HEADER
# include "lexer-tab.hh"
# include "parser-tab.hh"
#endif
namespace nix::lexer::internal {
void initLoc(YYLTYPE * loc);

25
src/libexpr/lexer.l
View file

@ -41,16 +41,18 @@ namespace nix {
// we make use of the fact that the parser receives a private copy of the input
// string and can munge around in it.
static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
// getting the position is expensive and thus it is implemented lazily.
static StringToken unescapeStr(char * const s, size_t length, std::function<Pos()> && pos)
{
char * result = s;
bool noNullByte = true;
char * t = s;
char c;
// the input string is terminated with *two* NULs, so we can safely take
// *one* character after the one being checked against.
while ((c = *s++)) {
for (size_t i = 0; i < length; t++) {
char c = s[i++];
noNullByte &= c != '\0';
if (c == '\\') {
c = *s++;
c = s[i++];
if (c == 'n') *t = '\n';
else if (c == 'r') *t = '\r';
else if (c == 't') *t = '\t';
@ -59,12 +61,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
else if (c == '\r') {
/* Normalise CR and CR/LF into LF. */
*t = '\n';
if (*s == '\n') s++; /* cr/lf */
if (s[i] == '\n') i++; /* cr/lf */
}
else *t = c;
t++;
}
return {result, size_t(t - result)};
if (!noNullByte) {
forceNoNullByte({s, size_t(t - s)}, std::move(pos));
}
return {s, size_t(t - s)};
}
static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos)
@ -175,7 +179,7 @@ or { return OR_KW; }
/* It is impossible to match strings ending with '$' with one
regex because trailing contexts are only valid at the end
of a rule. (A sane but undocumented limitation.) */
yylval->str = unescapeStr(state->symbols, yytext, yyleng);
yylval->str = unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; });
return STR;
}
<STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@ -191,6 +195,7 @@ or { return OR_KW; }
\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
yylval->str = {yytext, (size_t) yyleng, true};
forceNoNullByte(yylval->str, [&]() { return state->positions[CUR_POS]; });
return IND_STR;
}
<IND_STRING>\'\'\$ |
@ -203,7 +208,7 @@ or { return OR_KW; }
return IND_STR;
}
<IND_STRING>\'\'\\{ANY} {
yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2);
yylval->str = unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; });
return IND_STR;
}
<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }

14
src/libexpr/meson.build
View file

@ -4,8 +4,6 @@ project('nix-expr', 'cpp',
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
@ -14,7 +12,7 @@ project('nix-expr', 'cpp',
cxx = meson.get_compiler('cpp')
subdir('build-utils-meson/deps-lists')
subdir('nix-meson-build-support/deps-lists')
configdata = configuration_data()
@ -25,9 +23,7 @@ deps_public_maybe_subproject = [
dependency('nix-store'),
dependency('nix-fetchers'),
]
subdir('build-utils-meson/subprojects')
subdir('build-utils-meson/threads')
subdir('nix-meson-build-support/subprojects')
boost = dependency(
'boost',
@ -79,7 +75,7 @@ add_project_arguments(
language : 'cpp',
)
subdir('build-utils-meson/diagnostics')
subdir('nix-meson-build-support/common')
parser_tab = custom_target(
input : 'parser.y',
@ -123,7 +119,7 @@ lexer_tab = custom_target(
install_dir : get_option('includedir') / 'nix',
)
subdir('build-utils-meson/generate-header')
subdir('nix-meson-build-support/generate-header')
generated_headers = []
foreach header : [
@ -207,4 +203,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true)
libraries_private = []
subdir('build-utils-meson/export')
subdir('nix-meson-build-support/export')

1
src/libexpr/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

5
src/libexpr/nixexpr.hh
View file

@ -168,7 +168,7 @@ struct ExprVar : Expr
the set stored in the environment that is `level` levels up
from the current one.*/
Level level;
Displacement displ;
Displacement displ = 0;
ExprVar(Symbol name) : name(name) { };
ExprVar(const PosIdx & pos, Symbol name) : pos(pos), name(name) { };
@ -242,7 +242,7 @@ struct ExprAttrs : Expr
Kind kind;
Expr * e;
PosIdx pos;
Displacement displ; // displacement
Displacement displ = 0; // displacement
AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain)
: kind(kind), e(e), pos(pos) { };
AttrDef() { };
@ -468,6 +468,7 @@ struct ExprBlackHole : Expr
void show(const SymbolTable & symbols, std::ostream & str) const override {}
void eval(EvalState & state, Env & env, Value & v) override;
void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override {}
[[noreturn]] static void throwInfiniteRecursionError(EvalState & state, Value & v);
};
extern ExprBlackHole eBlackHole;

6
src/libexpr/package.nix
View file

@ -40,8 +40,8 @@ mkMesonLibrary (finalAttrs: {
workDir = ./.;
fileset = fileset.unions [
../../build-utils-meson
./build-utils-meson
../../nix-meson-build-support
./nix-meson-build-support
../../.version
./.version
./meson.build
@ -96,8 +96,6 @@ mkMesonLibrary (finalAttrs: {
# https://github.com/NixOS/nixpkgs/issues/86131.
BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
} // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
meta = {

108
src/libexpr/parser-state.hh
View file

@ -88,6 +88,7 @@ struct ParserState
void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc);
void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def);
Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
Expr * stripIndentation(const PosIdx pos,
std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
@ -120,64 +121,29 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const
// Checking attrPath validity.
// ===========================
for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
ExprAttrs * nested;
if (i->symbol) {
ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
if (j != attrs->attrs.end()) {
if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
if (!attrs2) dupAttr(attrPath, pos, j->second.pos);
attrs = attrs2;
} else
nested = dynamic_cast<ExprAttrs *>(j->second.e);
if (!nested) {
attrPath.erase(i + 1, attrPath.end());
dupAttr(attrPath, pos, j->second.pos);
}
} else {
ExprAttrs * nested = new ExprAttrs;
nested = new ExprAttrs;
attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
attrs = nested;
}
} else {
ExprAttrs *nested = new ExprAttrs;
nested = new ExprAttrs;
attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
attrs = nested;
}
attrs = nested;
}
// Expr insertion.
// ==========================
if (i->symbol) {
ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
if (j != attrs->attrs.end()) {
// This attr path is already defined. However, if both
// e and the expr pointed by the attr path are two attribute sets,
// we want to merge them.
// Otherwise, throw an error.
auto ae = dynamic_cast<ExprAttrs *>(e);
auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
if (jAttrs && ae) {
if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
for (auto & ad : ae->attrs) {
auto j2 = jAttrs->attrs.find(ad.first);
if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
dupAttr(ad.first, j2->second.pos, ad.second.pos);
jAttrs->attrs.emplace(ad.first, ad.second);
if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
from.displ += jAttrs->inheritFromExprs->size();
}
}
jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
if (ae->inheritFromExprs) {
jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
ae->inheritFromExprs->begin(), ae->inheritFromExprs->end());
}
} else {
dupAttr(attrPath, pos, j->second.pos);
}
} else {
// This attr path is not defined. Let's create it.
attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
e->setName(i->symbol);
}
addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos));
} else {
attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
}
@ -189,6 +155,60 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const
}
}
/**
* Precondition: attrPath is used for error messages and should already contain
* symbol as its last element.
*/
inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def)
{
ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol);
if (j != attrs->attrs.end()) {
// This attr path is already defined. However, if both
// e and the expr pointed by the attr path are two attribute sets,
// we want to merge them.
// Otherwise, throw an error.
auto ae = dynamic_cast<ExprAttrs *>(def.e);
auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
// N.B. In a world in which we are less bound by our past mistakes, we
// would also test that jAttrs and ae are not recursive. The effect of
// not doing so is that any `rec` marker on ae is discarded, and any
// `rec` marker on jAttrs will apply to the attributes in ae.
// See https://github.com/NixOS/nix/issues/9020.
if (jAttrs && ae) {
if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
for (auto & ad : ae->attrs) {
if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
from.displ += jAttrs->inheritFromExprs->size();
}
attrPath.emplace_back(AttrName(ad.first));
addAttr(jAttrs, attrPath, ad.first, std::move(ad.second));
attrPath.pop_back();
}
ae->attrs.clear();
jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(),
std::make_move_iterator(ae->dynamicAttrs.begin()),
std::make_move_iterator(ae->dynamicAttrs.end()));
ae->dynamicAttrs.clear();
if (ae->inheritFromExprs) {
jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
std::make_move_iterator(ae->inheritFromExprs->begin()),
std::make_move_iterator(ae->inheritFromExprs->end()));
ae->inheritFromExprs = nullptr;
}
} else {
dupAttr(attrPath, def.pos, j->second.pos);
}
} else {
// This attr path is not defined. Let's create it.
attrs->attrs.emplace(symbol, def);
def.e->setName(symbol);
}
}
inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
{
std::sort(formals->formals.begin(), formals->formals.end(),

78
src/libexpr/primops.cc
View file

@ -66,14 +66,12 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
ensureValid(b.drvPath->getBaseStorePath());
},
[&](const NixStringContextElem::Opaque & o) {
auto ctxS = store->printStorePath(o.path);
ensureValid(o.path);
if (maybePathsOut)
maybePathsOut->emplace(o.path);
},
[&](const NixStringContextElem::DrvDeep & d) {
/* Treat same as Opaque */
auto ctxS = store->printStorePath(d.drvPath);
ensureValid(d.drvPath);
if (maybePathsOut)
maybePathsOut->emplace(d.drvPath);
@ -121,11 +119,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow);
if (isIFD) {
for (auto & outputPath : outputsToCopyAndAllow) {
/* Add the output of this derivations to the allowed
paths. */
allowPath(outputPath);
}
/* Allow access to the output closures of this derivation. */
for (auto & outputPath : outputsToCopyAndAllow)
allowClosure(outputPath);
}
return res;
@ -724,7 +720,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a
/* Call the `operator' function with `e' as argument. */
Value newElements;
state.callFunction(*op->value, 1, &e, newElements, noPos);
state.callFunction(*op->value, {&e, 1}, newElements, noPos);
state.forceList(newElements, noPos, "while evaluating the return value of the `operator` passed to builtins.genericClosure");
/* Add the values returned by the operator to the work set. */
@ -1102,7 +1098,7 @@ static RegisterPrimOp primop_warn({
.name = "__warn",
.args = {"e1", "e2"},
.doc = R"(
Evaluate *e1*, which must be a string and print iton standard error as a warning.
Evaluate *e1*, which must be a string, and print it on standard error as a warning.
Then return *e2*.
This function is useful for non-critical situations where attention is advisable.
@ -1603,7 +1599,8 @@ static RegisterPrimOp primop_placeholder({
*************************************************************/
/* Convert the argument to a path. !!! obsolete? */
/* Convert the argument to a path and then to a string (confusing,
eh?). !!! obsolete? */
static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
NixStringContext context;
@ -2048,7 +2045,7 @@ static RegisterPrimOp primop_readFileType({
.args = {"p"},
.doc = R"(
Determine the directory entry type of a filesystem node, being
one of "directory", "regular", "symlink", or "unknown".
one of `"directory"`, `"regular"`, `"symlink"`, or `"unknown"`.
)",
.fun = prim_readFileType,
});
@ -2437,7 +2434,6 @@ static RegisterPrimOp primop_toFile({
bool EvalState::callPathFilter(
Value * filterFun,
const SourcePath & path,
std::string_view pathArg,
PosIdx pos)
{
auto st = path.lstat();
@ -2445,12 +2441,12 @@ bool EvalState::callPathFilter(
/* Call the filter function. The first argument is the path, the
second is a string indicating the type of the file. */
Value arg1;
arg1.mkString(pathArg);
arg1.mkString(path.path.abs());
// assert that type is not "unknown"
Value * args []{&arg1, fileTypeToString(*this, st.type)};
Value res;
callFunction(*filterFun, 2, args, res, pos);
callFunction(*filterFun, args, res, pos);
return forceBool(res, pos, "while evaluating the return value of the path filter function");
}
@ -2488,7 +2484,7 @@ static void addPath(
if (filterFun)
filter = std::make_unique<PathFilter>([&](const Path & p) {
auto p2 = CanonPath(p);
return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos);
return state.callPathFilter(filterFun, {path.accessor, p2}, pos);
});
std::optional<StorePath> expectedStorePath;
@ -2614,13 +2610,13 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
else
state.error<EvalError>(
"unsupported argument '%1%' to 'addPath'",
"unsupported argument '%1%' to 'builtins.path'",
state.symbols[attr.name]
).atPos(attr.pos).debugThrow();
}
if (!path)
state.error<EvalError>(
"missing required 'path' attribute in the first argument to builtins.path"
"missing required 'path' attribute in the first argument to 'builtins.path'"
).atPos(pos).debugThrow();
if (name.empty())
name = path->baseName();
@ -3261,23 +3257,19 @@ static RegisterPrimOp primop_isList({
.fun = prim_isList,
});
static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Value & v)
{
state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt");
if (n < 0 || (unsigned int) n >= list.listSize())
state.error<EvalError>(
"list index %1% is out of bounds",
n
).atPos(pos).debugThrow();
state.forceValue(*list.listElems()[n], pos);
v = *list.listElems()[n];
}
/* Return the n-1'th element of a list. */
static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value;
elemAt(state, pos, *args[0], elem, v);
NixInt::Inner n = state.forceInt(*args[1], pos, "while evaluating the second argument passed to 'builtins.elemAt'").value;
state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.elemAt'");
if (n < 0 || (unsigned int) n >= args[0]->listSize())
state.error<EvalError>(
"'builtins.elemAt' called with index %d on a list of size %d",
n,
args[0]->listSize()
).atPos(pos).debugThrow();
state.forceValue(*args[0]->listElems()[n], pos);
v = *args[0]->listElems()[n];
}
static RegisterPrimOp primop_elemAt({
@ -3293,7 +3285,13 @@ static RegisterPrimOp primop_elemAt({
/* Return the first element of a list. */
static void prim_head(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
elemAt(state, pos, *args[0], 0, v);
state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.head'");
if (args[0]->listSize() == 0)
state.error<EvalError>(
"'builtins.head' called on an empty list"
).atPos(pos).debugThrow();
state.forceValue(*args[0]->listElems()[0], pos);
v = *args[0]->listElems()[0];
}
static RegisterPrimOp primop_head({
@ -3312,9 +3310,9 @@ static RegisterPrimOp primop_head({
don't want to use it! */
static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail");
state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.tail'");
if (args[0]->listSize() == 0)
state.error<EvalError>("'tail' called on an empty list").atPos(pos).debugThrow();
state.error<EvalError>("'builtins.tail' called on an empty list").atPos(pos).debugThrow();
auto list = state.buildList(args[0]->listSize() - 1);
for (const auto & [n, v] : enumerate(list))
@ -3487,7 +3485,7 @@ static void prim_foldlStrict(EvalState & state, const PosIdx pos, Value * * args
for (auto [n, elem] : enumerate(args[2]->listItems())) {
Value * vs []{vCur, elem};
vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
state.callFunction(*args[0], 2, vs, *vCur, pos);
state.callFunction(*args[0], vs, *vCur, pos);
}
state.forceValue(v, pos);
} else {
@ -3637,7 +3635,7 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value
Value * vs[] = {a, b};
Value vBool;
state.callFunction(*args[0], 2, vs, vBool, noPos);
state.callFunction(*args[0], vs, vBool, noPos);
return state.forceBool(vBool, pos, "while evaluating the return value of the sorting function passed to builtins.sort");
};
@ -4061,7 +4059,7 @@ static RegisterPrimOp primop_toString({
});
/* `substring start len str' returns the substring of `str' starting
at character position `min(start, stringLength str)' inclusive and
at byte position `min(start, stringLength str)' inclusive and
ending at `min(start + len, stringLength str)'. `start' must be
non-negative. */
static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@ -4100,7 +4098,7 @@ static RegisterPrimOp primop_substring({
.name = "__substring",
.args = {"start", "len", "s"},
.doc = R"(
Return the substring of *s* from character position *start*
Return the substring of *s* from byte position *start*
(zero-based) up to but not including *start + len*. If *start* is
greater than the length of the string, an empty string is returned.
If *start + len* lies beyond the end of the string or *len* is `-1`,
@ -4385,7 +4383,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
for (auto i = begin; i != end; ++i) {
assert(idx <= 2 * len + 1 - 3);
auto match = *i;
const auto & match = *i;
// Add a string for non-matched characters.
list[idx++] = mkString(state, match.prefix());
@ -4937,7 +4935,7 @@ void EvalState::createBaseEnv()
/* Now that we've added all primops, sort the `builtins' set,
because attribute lookups expect it to be sorted. */
baseEnv.values[0]->payload.attrs->sort();
getBuiltins().payload.attrs->sort();
staticBaseEnv->sort();

2
src/libexpr/primops/context.cc
View file

@ -132,6 +132,8 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
},
[&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep {
/* Reuse original item because we want this to be idempotent. */
/* FIXME: Suspicious move out of const. This is actually a copy, so the comment
above does not make much sense. */
return std::move(c);
},
}, context.begin()->raw) }),

2
src/libexpr/primops/fetchClosure.cc
View file

@ -40,7 +40,7 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor
});
}
auto toPath = *toPathMaybe;
const auto & toPath = *toPathMaybe;
// check and return

7
src/libexpr/primops/fetchTree.cc
View file

@ -33,9 +33,8 @@ void emitTreeAttrs(
// FIXME: support arbitrary input attributes.
auto narHash = input.getNarHash();
assert(narHash);
attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
if (auto narHash = input.getNarHash())
attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
if (input.getType() == "git")
attrs.alloc("submodules").mkBool(
@ -183,7 +182,7 @@ static void fetchTree(
if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
input = lookupInRegistries(state.store, input).first;
if (state.settings.pureEval && !input.isLocked()) {
if (state.settings.pureEval && !input.isConsideredLocked(state.fetchSettings)) {
auto fetcher = "fetchTree";
if (params.isFetchGit)
fetcher = "fetchGit";

14
src/libexpr/primops/fromTOML.cc
View file

@ -28,8 +28,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
auto attrs = state.buildBindings(size);
for(auto & elem : table)
for(auto & elem : table) {
forceNoNullByte(elem.first);
visit(attrs.alloc(elem.first), elem.second);
}
v.mkAttrs(attrs);
}
@ -54,7 +56,11 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
v.mkFloat(toml::get<NixFloat>(t));
break;;
case toml::value_t::string:
v.mkString(toml::get<std::string>(t));
{
auto s = toml::get<std::string_view>(t);
forceNoNullByte(s);
v.mkString(s);
}
break;;
case toml::value_t::local_datetime:
case toml::value_t::offset_datetime:
@ -66,7 +72,9 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
attrs.alloc("_type").mkString("timestamp");
std::ostringstream s;
s << t;
attrs.alloc("value").mkString(toView(s));
auto str = toView(s);
forceNoNullByte(str);
attrs.alloc("value").mkString(str);
v.mkAttrs(attrs);
} else {
throw std::runtime_error("Dates and times are not supported");

1
src/libexpr/print-options.hh
View file

@ -5,6 +5,7 @@
*/
#include <limits>
#include <stddef.h>
namespace nix {

6
src/libexpr/value-to-json.cc
View file

@ -108,7 +108,11 @@ json printValueAsJSON(EvalState & state, bool strict,
void printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore)
{
str << printValueAsJSON(state, strict, v, pos, context, copyToStore);
try {
str << printValueAsJSON(state, strict, v, pos, context, copyToStore);
} catch (nlohmann::json::exception & e) {
throw JSONSerializationError("JSON serialization error: %s", e.what());
}
}
json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,

3
src/libexpr/value-to-json.hh
View file

@ -16,4 +16,7 @@ nlohmann::json printValueAsJSON(EvalState & state, bool strict,
void printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true);
MakeError(JSONSerializationError, Error);
}

2
src/libexpr/value.hh
View file

@ -510,4 +510,6 @@ typedef std::shared_ptr<Value *> RootValue;
RootValue allocRootValue(Value * v);
void forceNoNullByte(std::string_view s, std::function<Pos()> = nullptr);
}

1
src/libfetchers-tests/build-utils-meson
View file

@ -1 +0,0 @@
../../build-utils-meson

14
src/libfetchers-tests/meson.build
View file

@ -4,8 +4,6 @@ project('nix-fetchers-tests', 'cpp',
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
@ -14,7 +12,7 @@ project('nix-fetchers-tests', 'cpp',
cxx = meson.get_compiler('cpp')
subdir('build-utils-meson/deps-lists')
subdir('nix-meson-build-support/deps-lists')
deps_private_maybe_subproject = [
dependency('nix-store-test-support'),
@ -22,12 +20,10 @@ deps_private_maybe_subproject = [
]
deps_public_maybe_subproject = [
]
subdir('build-utils-meson/subprojects')
subdir('nix-meson-build-support/subprojects')
subdir('build-utils-meson/threads')
subdir('build-utils-meson/export-all-symbols')
subdir('build-utils-meson/windows-version')
subdir('nix-meson-build-support/export-all-symbols')
subdir('nix-meson-build-support/windows-version')
rapidcheck = dependency('rapidcheck')
deps_private += rapidcheck
@ -44,7 +40,7 @@ add_project_arguments(
language : 'cpp',
)
subdir('build-utils-meson/diagnostics')
subdir('nix-meson-build-support/common')
sources = files(
'public-key.cc',

1
src/libfetchers-tests/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

8
src/libfetchers-tests/package.nix
View file

@ -26,8 +26,8 @@ mkMesonExecutable (finalAttrs: {
workDir = ./.;
fileset = fileset.unions [
../../build-utils-meson
./build-utils-meson
../../nix-meson-build-support
./nix-meson-build-support
../../.version
./.version
./meson.build
@ -54,10 +54,6 @@ mkMesonExecutable (finalAttrs: {
mesonFlags = [
];
env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
passthru = {
tests = {
run = runCommand "${finalAttrs.pname}-run" {

1
src/libfetchers/build-utils-meson
View file

@ -1 +0,0 @@
../../build-utils-meson

2
src/libfetchers/cache.cc
View file

@ -36,7 +36,7 @@ struct CacheImpl : Cache
{
auto state(_state.lock());
auto dbPath = getCacheDir() + "/fetcher-cache-v2.sqlite";
auto dbPath = getCacheDir() + "/fetcher-cache-v3.sqlite";
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath);

16
src/libfetchers/fetch-settings.hh
View file

@ -70,6 +70,22 @@ struct Settings : public Config
Setting<bool> warnDirty{this, true, "warn-dirty",
"Whether to warn about dirty Git/Mercurial trees."};
Setting<bool> allowDirtyLocks{
this,
false,
"allow-dirty-locks",
R"(
Whether to allow dirty inputs (such as dirty Git workdirs)
to be locked via their NAR hash. This is generally bad
practice since Nix has no way to obtain such inputs if they
are subsequently modified. Therefore lock files with dirty
locks should generally only be used for local testing, and
should not be pushed to other users.
)",
{},
true,
Xp::Flakes};
Setting<bool> trustTarballsFromGitForges{
this, true, "trust-tarballs-from-git-forges",
R"(

2
src/libfetchers/fetch-to-store.cc
View file

@ -44,6 +44,8 @@ StorePath fetchToStore(
: store.addToStore(
name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
if (cacheKey && mode == FetchMode::Copy)
fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);

33
src/libfetchers/fetchers.cc
View file

@ -4,6 +4,7 @@
#include "fetch-to-store.hh"
#include "json-utils.hh"
#include "store-path-accessor.hh"
#include "fetch-settings.hh"
#include <nlohmann/json.hpp>
@ -66,7 +67,7 @@ Input Input::fromURL(
}
}
throw Error("input '%s' is unsupported", url.url);
throw Error("input '%s' is unsupported", url);
}
Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
@ -113,7 +114,15 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
std::optional<std::string> Input::getFingerprint(ref<Store> store) const
{
return scheme ? scheme->getFingerprint(store, *this) : std::nullopt;
if (!scheme) return std::nullopt;
if (cachedFingerprint) return *cachedFingerprint;
auto fingerprint = scheme->getFingerprint(store, *this);
cachedFingerprint = fingerprint;
return fingerprint;
}
ParsedURL Input::toURL() const
@ -146,11 +155,23 @@ bool Input::isLocked() const
return scheme && scheme->isLocked(*this);
}
bool Input::isConsideredLocked(
const Settings & settings) const
{
return isLocked() || (settings.allowDirtyLocks && getNarHash());
}
bool Input::isFinal() const
{
return maybeGetBoolAttr(attrs, "__final").value_or(false);
}
std::optional<std::string> Input::isRelative() const
{
assert(scheme);
return scheme->isRelative(*this);
}
Attrs Input::toAttrs() const
{
return attrs;
@ -307,7 +328,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
auto accessor = makeStorePathAccessor(store, storePath);
accessor->fingerprint = scheme->getFingerprint(store, *this);
accessor->fingerprint = getFingerprint(store);
return {accessor, *this};
} catch (Error & e) {
@ -318,7 +339,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
auto [accessor, result] = scheme->getAccessor(store, *this);
assert(!accessor->fingerprint);
accessor->fingerprint = scheme->getFingerprint(store, result);
accessor->fingerprint = result.getFingerprint(store);
return {accessor, std::move(result)};
}
@ -337,7 +358,7 @@ void Input::clone(const Path & destDir) const
scheme->clone(*this, destDir);
}
std::optional<Path> Input::getSourcePath() const
std::optional<std::filesystem::path> Input::getSourcePath() const
{
assert(scheme);
return scheme->getSourcePath(*this);
@ -440,7 +461,7 @@ Input InputScheme::applyOverrides(
return input;
}
std::optional<Path> InputScheme::getSourcePath(const Input & input) const
std::optional<std::filesystem::path> InputScheme::getSourcePath(const Input & input) const
{
return {};
}

31
src/libfetchers/fetchers.hh
View file

@ -42,9 +42,9 @@ struct Input
Attrs attrs;
/**
* path of the parent of this input, used for relative path resolution
* Cached result of getFingerprint().
*/
std::optional<Path> parent;
mutable std::optional<std::optional<std::string>> cachedFingerprint;
public:
/**
@ -90,6 +90,21 @@ public:
*/
bool isLocked() const;
/**
* Return whether the input is either locked, or, if
* `allow-dirty-locks` is enabled, it has a NAR hash. In the
* latter case, we can verify the input but we may not be able to
* fetch it from anywhere.
*/
bool isConsideredLocked(
const Settings & settings) const;
/**
* Only for relative path flakes, i.e. 'path:./foo', returns the
* relative path, i.e. './foo'.
*/
std::optional<std::string> isRelative() const;
/**
* Return whether this is a "final" input, meaning that fetching
* it will not add, remove or change any attributes. (See
@ -104,6 +119,11 @@ public:
bool operator ==(const Input & other) const noexcept;
bool operator <(const Input & other) const
{
return attrs < other.attrs;
}
bool contains(const Input & other) const;
/**
@ -144,7 +164,7 @@ public:
void clone(const Path & destDir) const;
std::optional<Path> getSourcePath() const;
std::optional<std::filesystem::path> getSourcePath() const;
/**
* Write a file to this input, for input types that support
@ -227,7 +247,7 @@ struct InputScheme
virtual void clone(const Input & input, const Path & destDir) const;
virtual std::optional<Path> getSourcePath(const Input & input) const;
virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const;
virtual void putFile(
const Input & input,
@ -250,6 +270,9 @@ struct InputScheme
virtual bool isLocked(const Input & input) const
{ return false; }
virtual std::optional<std::string> isRelative(const Input & input) const
{ return std::nullopt; }
};
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

19
src/libfetchers/git-utils.cc
View file

@ -5,6 +5,7 @@
#include "signals.hh"
#include "users.hh"
#include "fs-sink.hh"
#include "sync.hh"
#include <git2/attr.h>
#include <git2/blob.h>
@ -437,7 +438,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
{
if (!(statusFlags & GIT_STATUS_INDEX_DELETED) &&
!(statusFlags & GIT_STATUS_WT_DELETED))
{
info.files.insert(CanonPath(path));
if (statusFlags != GIT_STATUS_CURRENT)
info.dirtyFiles.insert(CanonPath(path));
} else
info.deletedFiles.insert(CanonPath(path));
if (statusFlags != GIT_STATUS_CURRENT)
info.isDirty = true;
return 0;
@ -1262,4 +1268,17 @@ ref<GitRepo> getTarballCache()
return GitRepo::openRepo(repoDir, true, true);
}
GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path)
{
static Sync<std::map<std::filesystem::path, WorkdirInfo>> _cache;
{
auto cache(_cache.lock());
auto i = cache->find(path);
if (i != cache->end()) return i->second;
}
auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo();
_cache.lock()->emplace(path, workdirInfo);
return workdirInfo;
}
}

8
src/libfetchers/git-utils.hh
View file

@ -59,12 +59,20 @@ struct GitRepo
modified or added, but excluding deleted files. */
std::set<CanonPath> files;
/* All modified or added files. */
std::set<CanonPath> dirtyFiles;
/* The deleted files. */
std::set<CanonPath> deletedFiles;
/* The submodules listed in .gitmodules of this workdir. */
std::vector<Submodule> submodules;
};
virtual WorkdirInfo getWorkdirInfo() = 0;
static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path);
/* Get the ref that HEAD points to. */
virtual std::optional<std::string> getWorkdirRef() = 0;

162
src/libfetchers/git.cc
View file

@ -15,6 +15,7 @@
#include "finally.hh"
#include "fetch-settings.hh"
#include "json-utils.hh"
#include "archive.hh"
#include <regex>
#include <string.h>
@ -296,7 +297,7 @@ struct GitInputScheme : InputScheme
Strings args = {"clone"};
args.push_back(repoInfo.url);
args.push_back(repoInfo.locationToArg());
if (auto ref = input.getRef()) {
args.push_back("--branch");
@ -310,11 +311,9 @@ struct GitInputScheme : InputScheme
runProgram("git", true, args, {}, true);
}
std::optional<Path> getSourcePath(const Input & input) const override
std::optional<std::filesystem::path> getSourcePath(const Input & input) const override
{
auto repoInfo = getRepoInfo(input);
if (repoInfo.isLocal) return repoInfo.url;
return std::nullopt;
return getRepoInfo(input).getPath();
}
void putFile(
@ -324,14 +323,15 @@ struct GitInputScheme : InputScheme
std::optional<std::string> commitMsg) const override
{
auto repoInfo = getRepoInfo(input);
if (!repoInfo.isLocal)
auto repoPath = repoInfo.getPath();
if (!repoPath)
throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
writeFile((CanonPath(repoInfo.url) / path).abs(), contents);
writeFile(*repoPath / path.rel(), contents);
auto result = runProgram(RunOptions {
.program = "git",
.args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
.args = {"-C", *repoPath, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
});
auto exitCode =
#ifndef WIN32 // TODO abstract over exit status handling on Windows
@ -344,7 +344,7 @@ struct GitInputScheme : InputScheme
if (exitCode != 0) {
// The path is not `.gitignore`d, we can add the file.
runProgram("git", true,
{ "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
{ "-C", *repoPath, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
if (commitMsg) {
@ -352,7 +352,7 @@ struct GitInputScheme : InputScheme
logger->pause();
Finally restoreLogger([]() { logger->resume(); });
runProgram("git", true,
{ "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" },
{ "-C", *repoPath, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" },
*commitMsg);
}
}
@ -360,24 +360,41 @@ struct GitInputScheme : InputScheme
struct RepoInfo
{
/* Whether this is a local, non-bare repository. */
bool isLocal = false;
/* Either the path of the repo (for local, non-bare repos), or
the URL (which is never a `file` URL). */
std::variant<std::filesystem::path, ParsedURL> location;
/* Working directory info: the complete list of files, and
whether the working directory is dirty compared to HEAD. */
GitRepo::WorkdirInfo workdirInfo;
/* URL of the repo, or its path if isLocal. Never a `file` URL. */
std::string url;
std::string locationToArg() const
{
return std::visit(
overloaded {
[&](const std::filesystem::path & path)
{ return path.string(); },
[&](const ParsedURL & url)
{ return url.to_string(); }
}, location);
}
std::optional<std::filesystem::path> getPath() const
{
if (auto path = std::get_if<std::filesystem::path>(&location))
return *path;
else
return std::nullopt;
}
void warnDirty(const Settings & settings) const
{
if (workdirInfo.isDirty) {
if (!settings.allowDirty)
throw Error("Git tree '%s' is dirty", url);
throw Error("Git tree '%s' is dirty", locationToArg());
if (settings.warnDirty)
warn("Git tree '%s' is dirty", url);
warn("Git tree '%s' is dirty", locationToArg());
}
}
@ -424,13 +441,31 @@ struct GitInputScheme : InputScheme
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
repoInfo.url = repoInfo.isLocal ? url.path : url.base;
//
// FIXME: here we turn a possibly relative path into an absolute path.
// This allows relative git flake inputs to be resolved against the
// **current working directory** (as in POSIX), which tends to work out
// ok in the context of flakes, but is the wrong behavior,
// as it should resolve against the flake.nix base directory instead.
//
// See: https://discourse.nixos.org/t/57783 and #9708
//
if (url.scheme == "file" && !forceHttp && !isBareRepository) {
if (!isAbsolute(url.path)) {
warn(
"Fetching Git repository '%s', which uses a path relative to the current directory. "
"This is not supported and will stop working in a future release. "
"See https://github.com/NixOS/nix/issues/12281 for details.",
url);
}
repoInfo.location = std::filesystem::absolute(url.path);
} else
repoInfo.location = url;
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
if (!input.getRef() && !input.getRev() && repoInfo.isLocal)
repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo();
if (auto repoPath = repoInfo.getPath(); !input.getRef() && !input.getRev() && repoPath)
repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(*repoPath);
return repoInfo;
}
@ -460,7 +495,7 @@ struct GitInputScheme : InputScheme
if (auto revCountAttrs = cache->lookup(key))
return getIntAttr(*revCountAttrs, "revCount");
Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url));
Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.locationToArg()));
auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev);
@ -471,11 +506,15 @@ struct GitInputScheme : InputScheme
std::string getDefaultRef(const RepoInfo & repoInfo) const
{
auto head = repoInfo.isLocal
? GitRepo::openRepo(repoInfo.url)->getWorkdirRef()
: readHeadCached(repoInfo.url);
auto head = std::visit(
overloaded {
[&](const std::filesystem::path & path)
{ return GitRepo::openRepo(path)->getWorkdirRef(); },
[&](const ParsedURL & url)
{ return readHeadCached(url.to_string()); }
}, repoInfo.location);
if (!head) {
warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url);
warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg());
return "master";
}
return *head;
@ -514,20 +553,19 @@ struct GitInputScheme : InputScheme
auto origRev = input.getRev();
std::string name = input.getName();
auto originalRef = input.getRef();
auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
input.attrs.insert_or_assign("ref", ref);
Path repoDir;
if (repoInfo.isLocal) {
repoDir = repoInfo.url;
if (auto repoPath = repoInfo.getPath()) {
repoDir = *repoPath;
if (!input.getRev())
input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
} else {
Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input));
auto repoUrl = std::get<ParsedURL>(repoInfo.location);
Path cacheDir = getCachePath(repoUrl.to_string(), getShallowAttr(input));
repoDir = cacheDir;
repoInfo.gitDir = ".";
@ -537,7 +575,7 @@ struct GitInputScheme : InputScheme
auto repo = GitRepo::openRepo(cacheDir, true, true);
// We need to set the origin so resolving submodule URLs works
repo->setRemote("origin", repoInfo.url);
repo->setRemote("origin", repoUrl.to_string());
Path localRefFile =
ref.compare(0, 5, "refs/") == 0
@ -576,11 +614,11 @@ struct GitInputScheme : InputScheme
? ref
: "refs/heads/" + ref;
repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input));
repo->fetch(repoUrl.to_string(), fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input));
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
logError(e.info());
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url);
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.locationToArg());
}
try {
@ -589,8 +627,8 @@ struct GitInputScheme : InputScheme
} catch (Error & e) {
warn("could not update mtime for file '%s': %s", localRefFile, e.info().msg);
}
if (!originalRef && !storeCachedHead(repoInfo.url, ref))
warn("could not update cached head '%s' for '%s'", ref, repoInfo.url);
if (!originalRef && !storeCachedHead(repoUrl.to_string(), ref))
warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
}
if (auto rev = input.getRev()) {
@ -602,8 +640,7 @@ struct GitInputScheme : InputScheme
"allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
rev->gitRev(),
ref,
repoInfo.url
);
repoInfo.locationToArg());
} else
input.attrs.insert_or_assign("rev", repo->resolveRef(ref).gitRev());
@ -615,7 +652,7 @@ struct GitInputScheme : InputScheme
auto isShallow = repo->isShallow();
if (isShallow && !getShallowAttr(input))
throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url);
throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.locationToArg());
// FIXME: check whether rev is an ancestor of ref?
@ -630,7 +667,7 @@ struct GitInputScheme : InputScheme
infoAttrs.insert_or_assign("revCount",
getRevCount(repoInfo, repoDir, rev));
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url);
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
verifyCommit(input, repo);
@ -684,21 +721,23 @@ struct GitInputScheme : InputScheme
RepoInfo & repoInfo,
Input && input) const
{
auto repoPath = repoInfo.getPath().value();
if (getSubmodulesAttr(input))
/* Create mountpoints for the submodules. */
for (auto & submodule : repoInfo.workdirInfo.submodules)
repoInfo.workdirInfo.files.insert(submodule.path);
auto repo = GitRepo::openRepo(repoInfo.url, false, false);
auto repo = GitRepo::openRepo(repoPath, false, false);
auto exportIgnore = getExportIgnoreAttr(input);
ref<SourceAccessor> accessor =
repo->getAccessor(repoInfo.workdirInfo,
exportIgnore,
makeNotAllowedError(repoInfo.url));
makeNotAllowedError(repoInfo.locationToArg()));
accessor->setPathDisplay(repoInfo.url);
accessor->setPathDisplay(repoInfo.locationToArg());
/* If the repo has submodules, return a mounted input accessor
consisting of the accessor for the top-level repo and the
@ -707,10 +746,10 @@ struct GitInputScheme : InputScheme
std::map<CanonPath, nix::ref<SourceAccessor>> mounts;
for (auto & submodule : repoInfo.workdirInfo.submodules) {
auto submodulePath = CanonPath(repoInfo.url) / submodule.path;
auto submodulePath = repoPath / submodule.path.rel();
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", submodulePath.abs());
attrs.insert_or_assign("url", submodulePath.string());
attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
attrs.insert_or_assign("submodules", Explicit<bool>{ true });
// TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out
@ -734,7 +773,7 @@ struct GitInputScheme : InputScheme
}
if (!repoInfo.workdirInfo.isDirty) {
auto repo = GitRepo::openRepo(repoInfo.url);
auto repo = GitRepo::openRepo(repoPath);
if (auto ref = repo->getWorkdirRef())
input.attrs.insert_or_assign("ref", *ref);
@ -744,7 +783,7 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign("rev", rev.gitRev());
input.attrs.insert_or_assign("revCount",
rev == nullRev ? 0 : getRevCount(repoInfo, repoInfo.url, rev));
rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev));
verifyCommit(input, repo);
} else {
@ -763,7 +802,7 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign(
"lastModified",
repoInfo.workdirInfo.headRev
? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev)
? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
: 0);
return {accessor, std::move(input)};
@ -786,7 +825,7 @@ struct GitInputScheme : InputScheme
}
auto [accessor, final] =
input.getRef() || input.getRev() || !repoInfo.isLocal
input.getRef() || input.getRev() || !repoInfo.getPath()
? getAccessorFromCommit(store, repoInfo, std::move(input))
: getAccessorFromWorkdir(store, repoInfo, std::move(input));
@ -795,10 +834,33 @@ struct GitInputScheme : InputScheme
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
auto makeFingerprint = [&](const Hash & rev)
{
return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "");
};
if (auto rev = input.getRev())
return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "");
else
return makeFingerprint(*rev);
else {
auto repoInfo = getRepoInfo(input);
if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) {
/* Calculate a fingerprint that takes into account the
deleted and modified/added files. */
HashSink hashSink{HashAlgorithm::SHA512};
for (auto & file : repoInfo.workdirInfo.dirtyFiles) {
writeString("modified:", hashSink);
writeString(file.abs(), hashSink);
dumpPath(*repoPath / file.rel(), hashSink);
}
for (auto & file : repoInfo.workdirInfo.deletedFiles) {
writeString("deleted:", hashSink);
writeString(file.abs(), hashSink);
}
return makeFingerprint(*repoInfo.workdirInfo.headRev)
+ ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false);
}
return std::nullopt;
}
}
bool isLocked(const Input & input) const override

16
src/libfetchers/github.cc
View file

@ -50,7 +50,7 @@ struct GitArchiveInputScheme : InputScheme
else if (std::regex_match(path[2], refRegex))
ref = path[2];
else
throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url, path[2]);
} else if (size > 3) {
std::string rs;
for (auto i = std::next(path.begin(), 2); i != path.end(); i++) {
@ -63,34 +63,34 @@ struct GitArchiveInputScheme : InputScheme
if (std::regex_match(rs, refRegex)) {
ref = rs;
} else {
throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs);
throw BadURL("in URL '%s', '%s' is not a branch/tag name", url, rs);
}
} else if (size < 2)
throw BadURL("URL '%s' is invalid", url.url);
throw BadURL("URL '%s' is invalid", url);
for (auto &[name, value] : url.query) {
if (name == "rev") {
if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url.url);
throw BadURL("URL '%s' contains multiple commit hashes", url);
rev = Hash::parseAny(value, HashAlgorithm::SHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
throw BadURL("URL '%s' contains an invalid branch/tag name", url);
if (ref)
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
throw BadURL("URL '%s' contains multiple branch/tag names", url);
ref = value;
}
else if (name == "host") {
if (!std::regex_match(value, hostRegex))
throw BadURL("URL '%s' contains an invalid instance host", url.url);
throw BadURL("URL '%s' contains an invalid instance host", url);
host_url = value;
}
// FIXME: barf on unsupported attributes
}
if (ref && rev)
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev());
Input input{settings};
input.attrs.insert_or_assign("type", std::string { schemeName() });

8
src/libfetchers/indirect.cc
View file

@ -26,16 +26,16 @@ struct IndirectInputScheme : InputScheme
else if (std::regex_match(path[1], refRegex))
ref = path[1];
else
throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url, path[1]);
} else if (path.size() == 3) {
if (!std::regex_match(path[1], refRegex))
throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url, path[1]);
ref = path[1];
if (!std::regex_match(path[2], revRegex))
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url, path[2]);
rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);
throw BadURL("GitHub URL '%s' is invalid", url);
std::string id = path[0];
if (!std::regex_match(id, flakeRegex))

4
src/libfetchers/mercurial.cc
View file

@ -126,7 +126,7 @@ struct MercurialInputScheme : InputScheme
return res;
}
std::optional<Path> getSourcePath(const Input & input) const override
std::optional<std::filesystem::path> getSourcePath(const Input & input) const override
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
if (url.scheme == "file" && !input.getRef() && !input.getRev())
@ -161,7 +161,7 @@ struct MercurialInputScheme : InputScheme
{
auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file";
return {isLocal, isLocal ? url.path : url.base};
return {isLocal, isLocal ? url.path : url.to_string()};
}
StorePath fetchToStore(ref<Store> store, Input & input) const

18
src/libfetchers/meson.build
View file

@ -4,8 +4,6 @@ project('nix-fetchers', 'cpp',
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
@ -14,7 +12,7 @@ project('nix-fetchers', 'cpp',
cxx = meson.get_compiler('cpp')
subdir('build-utils-meson/deps-lists')
subdir('nix-meson-build-support/deps-lists')
configdata = configuration_data()
@ -24,9 +22,7 @@ deps_public_maybe_subproject = [
dependency('nix-util'),
dependency('nix-store'),
]
subdir('build-utils-meson/subprojects')
subdir('build-utils-meson/threads')
subdir('nix-meson-build-support/subprojects')
nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
deps_public += nlohmann_json
@ -43,7 +39,7 @@ add_project_arguments(
language : 'cpp',
)
subdir('build-utils-meson/diagnostics')
subdir('nix-meson-build-support/common')
sources = files(
'attrs.cc',
@ -52,15 +48,15 @@ sources = files(
'fetch-to-store.cc',
'fetchers.cc',
'filtering-source-accessor.cc',
'git.cc',
'git-utils.cc',
'git.cc',
'github.cc',
'indirect.cc',
'mercurial.cc',
'mounted-source-accessor.cc',
'path.cc',
'store-path-accessor.cc',
'registry.cc',
'store-path-accessor.cc',
'tarball.cc',
)
@ -71,10 +67,10 @@ headers = files(
'cache.hh',
'fetch-settings.hh',
'fetch-to-store.hh',
'fetchers.hh',
'filtering-source-accessor.hh',
'git-utils.hh',
'mounted-source-accessor.hh',
'fetchers.hh',
'registry.hh',
'store-path-accessor.hh',
'tarball.hh',
@ -92,4 +88,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true)
libraries_private = []
subdir('build-utils-meson/export')
subdir('nix-meson-build-support/export')

1
src/libfetchers/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

9
src/libfetchers/package.nix
View file

@ -1,5 +1,4 @@
{ lib
, stdenv
, mkMesonLibrary
, nix-util
@ -22,8 +21,8 @@ mkMesonLibrary (finalAttrs: {
workDir = ./.;
fileset = fileset.unions [
../../build-utils-meson
./build-utils-meson
../../nix-meson-build-support
./nix-meson-build-support
../../.version
./.version
./meson.build
@ -49,10 +48,6 @@ mkMesonLibrary (finalAttrs: {
echo ${version} > ../../.version
'';
env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
meta = {
platforms = lib.platforms.unix ++ lib.platforms.windows;
};

49
src/libfetchers/path.cc
View file

@ -14,7 +14,7 @@ struct PathInputScheme : InputScheme
if (url.scheme != "path") return {};
if (url.authority && *url.authority != "")
throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority);
Input input{settings};
input.attrs.insert_or_assign("type", "path");
@ -27,10 +27,10 @@ struct PathInputScheme : InputScheme
if (auto n = string2Int<uint64_t>(value))
input.attrs.insert_or_assign(name, *n);
else
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
throw Error("path URL '%s' has invalid parameter '%s'", url, name);
}
else
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
throw Error("path URL '%s' has unsupported parameter '%s'", url, name);
return input;
}
@ -80,9 +80,9 @@ struct PathInputScheme : InputScheme
};
}
std::optional<Path> getSourcePath(const Input & input) const override
std::optional<std::filesystem::path> getSourcePath(const Input & input) const override
{
return getStrAttr(input.attrs, "path");
return getAbsPath(input);
}
void putFile(
@ -91,13 +91,13 @@ struct PathInputScheme : InputScheme
std::string_view contents,
std::optional<std::string> commitMsg) const override
{
writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents);
writeFile(getAbsPath(input) / path.rel(), contents);
}
std::optional<std::string> isRelative(const Input & input) const
std::optional<std::string> isRelative(const Input & input) const override
{
auto path = getStrAttr(input.attrs, "path");
if (hasPrefix(path, "/"))
if (isAbsolute(path))
return std::nullopt;
else
return path;
@ -108,12 +108,12 @@ struct PathInputScheme : InputScheme
return (bool) input.getNarHash();
}
CanonPath getAbsPath(const Input & input) const
std::filesystem::path getAbsPath(const Input & input) const
{
auto path = getStrAttr(input.attrs, "path");
if (path[0] == '/')
return CanonPath(path);
if (isAbsolute(path))
return canonPath(path);
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
}
@ -121,31 +121,14 @@ struct PathInputScheme : InputScheme
std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
{
Input input(_input);
std::string absPath;
auto path = getStrAttr(input.attrs, "path");
if (path[0] != '/') {
if (!input.parent)
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
auto absPath = getAbsPath(input);
auto parent = canonPath(*input.parent);
// the path isn't relative, prefix it
absPath = nix::absPath(path, parent);
// for security, ensure that if the parent is a store path, it's inside it
if (store->isInStore(parent)) {
auto storePath = store->printStorePath(store->toStorePath(parent).first);
if (!isDirOrInDir(absPath, storePath))
throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
}
} else
absPath = path;
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath));
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to the store", absPath));
// FIXME: check whether access to 'path' is allowed.
auto storePath = store->maybeParseStorePath(absPath);
auto storePath = store->maybeParseStorePath(absPath.string());
if (storePath)
store->addTempRoot(*storePath);
@ -154,7 +137,7 @@ struct PathInputScheme : InputScheme
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
// FIXME: try to substitute storePath.
auto src = sinkToSource([&](Sink & sink) {
mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter);
mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter);
});
storePath = store->addToStoreFromDump(*src, "source");
}
@ -176,7 +159,7 @@ struct PathInputScheme : InputScheme
store object and the subpath. */
auto path = getAbsPath(input);
try {
auto [storePath, subPath] = store->toStorePath(path.abs());
auto [storePath, subPath] = store->toStorePath(path.string());
auto info = store->queryPathInfo(storePath);
return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath);
} catch (Error &) {

15
src/libfetchers/registry.cc
View file

@ -94,12 +94,9 @@ void Registry::add(
void Registry::remove(const Input & input)
{
// FIXME: use C++20 std::erase.
for (auto i = entries.begin(); i != entries.end(); )
if (i->from == input)
i = entries.erase(i);
else
++i;
entries.erase(
std::remove_if(entries.begin(), entries.end(), [&](const Entry & entry) { return entry.from == input; }),
entries.end());
}
static Path getSystemRegistryPath()
@ -156,7 +153,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, re
return std::make_shared<Registry>(settings, Registry::Global); // empty registry
}
if (!hasPrefix(path, "/")) {
if (!isAbsolute(path)) {
auto storePath = downloadFile(store, path, "flake-registry.json").storePath;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
@ -181,7 +178,8 @@ Registries getRegistries(const Settings & settings, ref<Store> store)
std::pair<Input, Attrs> lookupInRegistries(
ref<Store> store,
const Input & _input)
const Input & _input,
const RegistryFilter & filter)
{
Attrs extraAttrs;
int n = 0;
@ -193,6 +191,7 @@ std::pair<Input, Attrs> lookupInRegistries(
if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
for (auto & registry : getRegistries(*input.settings, store)) {
if (filter && !filter(registry->type)) continue;
// FIXME: O(n)
for (auto & entry : registry->entries) {
if (entry.exact) {

9
src/libfetchers/registry.hh
View file

@ -65,8 +65,15 @@ void overrideRegistry(
const Input & to,
const Attrs & extraAttrs);
using RegistryFilter = std::function<bool(Registry::RegistryType)>;
/**
* Rewrite a flakeref using the registries. If `filter` is set, only
* use the registries for which the filter function returns true.
*/
std::pair<Input, Attrs> lookupInRegistries(
ref<Store> store,
const Input & input);
const Input & input,
const RegistryFilter & filter = {});
}

1
src/libflake-c/.version Symbolic link
View file

@ -0,0 +1 @@
../../.version

91
src/libflake-c/meson.build Normal file
View file

@ -0,0 +1,91 @@
project('nix-flake-c', 'cpp',
version : files('.version'),
default_options : [
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
license : 'LGPL-2.1-or-later',
)
cxx = meson.get_compiler('cpp')
subdir('nix-meson-build-support/deps-lists')
configdata = configuration_data()
deps_private_maybe_subproject = [
dependency('nix-util'),
dependency('nix-store'),
dependency('nix-expr'),
dependency('nix-flake'),
]
deps_public_maybe_subproject = [
dependency('nix-util-c'),
dependency('nix-store-c'),
dependency('nix-expr-c'),
]
subdir('nix-meson-build-support/subprojects')
# TODO rename, because it will conflict with downstream projects
configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
config_h = configure_file(
configuration : configdata,
output : 'config-flake.h',
)
add_project_arguments(
# TODO(Qyriad): Yes this is how the autoconf+Make system did it.
# It would be nice for our headers to be idempotent instead.
# From C++ libraries, only for internals
'-include', 'config-util.hh',
'-include', 'config-store.hh',
'-include', 'config-expr.hh',
# not generated (yet?)
# '-include', 'config-flake.hh',
# From C libraries, for our public, installed headers too
'-include', 'config-util.h',
'-include', 'config-store.h',
'-include', 'config-expr.h',
'-include', 'config-flake.h',
language : 'cpp',
)
subdir('nix-meson-build-support/common')
sources = files(
'nix_api_flake.cc',
)
include_dirs = [include_directories('.')]
headers = [config_h] + files(
'nix_api_flake.h',
)
# TODO move this header to libexpr, maybe don't use it in tests?
headers += files('nix_api_flake.h')
subdir('nix-meson-build-support/export-all-symbols')
subdir('nix-meson-build-support/windows-version')
this_library = library(
'nixflakec',
sources,
dependencies : deps_public + deps_private + deps_other,
include_directories : include_dirs,
link_args: linker_export_flags,
prelink : true, # For C++ static initializers
install : true,
)
install_headers(headers, subdir : 'nix', preserve_path : true)
libraries_private = []
subdir('nix-meson-build-support/export')

1
src/libflake-c/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

32
src/libflake-c/nix_api_flake.cc Normal file
View file

@ -0,0 +1,32 @@
#include "nix_api_flake.h"
#include "nix_api_flake_internal.hh"
#include "nix_api_util_internal.h"
#include "flake/flake.hh"
nix_flake_settings * nix_flake_settings_new(nix_c_context * context)
{
try {
auto settings = nix::make_ref<nix::flake::Settings>();
return new nix_flake_settings{settings};
}
NIXC_CATCH_ERRS_NULL
}
void nix_flake_settings_free(nix_flake_settings * settings)
{
delete settings;
}
nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings)
{
static std::shared_ptr<nix::flake::Settings> registeredSettings;
try {
if (registeredSettings)
throw nix::Error("nix_flake_init_global already initialized");
registeredSettings = settings->settings;
nix::flake::initLib(*registeredSettings);
}
NIXC_CATCH_ERRS
}

46
src/libflake-c/nix_api_flake.h Normal file
View file

@ -0,0 +1,46 @@
#ifndef NIX_API_FLAKE_H
#define NIX_API_FLAKE_H
/** @defgroup libflake libflake
* @brief Bindings to the Nix Flakes library
*
* @{
*/
/** @file
* @brief Main entry for the libflake C bindings
*/
#include "nix_api_store.h"
#include "nix_api_util.h"
#include "nix_api_expr.h"
#ifdef __cplusplus
extern "C" {
#endif
// cffi start
typedef struct nix_flake_settings nix_flake_settings;
// Function prototypes
/**
* Create a nix_flake_settings initialized with default values.
* @param[out] context Optional, stores error information
* @return A new nix_flake_settings or NULL on failure.
* @see nix_flake_settings_free
*/
nix_flake_settings * nix_flake_settings_new(nix_c_context * context);
/**
* @brief Release the resources associated with a nix_flake_settings.
*/
void nix_flake_settings_free(nix_flake_settings * settings);
/**
* @brief Register Flakes support process-wide.
*/
nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings);
#ifdef __cplusplus
} // extern "C"
#endif
#endif

9
src/libflake-c/nix_api_flake_internal.hh Normal file
View file

@ -0,0 +1,9 @@
#pragma once
#include "ref.hh"
#include "flake/settings.hh"
struct nix_flake_settings
{
nix::ref<nix::flake::Settings> settings;
};

55
src/libflake-c/package.nix Normal file
View file

@ -0,0 +1,55 @@
{ lib
, mkMesonLibrary
, nix-store-c
, nix-expr-c
, nix-flake
# Configuration Options
, version
}:
let
inherit (lib) fileset;
in
mkMesonLibrary (finalAttrs: {
pname = "nix-flake-c";
inherit version;
workDir = ./.;
fileset = fileset.unions [
../../nix-meson-build-support
./nix-meson-build-support
../../.version
./.version
./meson.build
# ./meson.options
(fileset.fileFilter (file: file.hasExt "cc") ./.)
(fileset.fileFilter (file: file.hasExt "hh") ./.)
(fileset.fileFilter (file: file.hasExt "h") ./.)
];
propagatedBuildInputs = [
nix-expr-c
nix-store-c
nix-flake
];
preConfigure =
# "Inline" .version so it's not a symlink, and includes the suffix.
# Do the meson utils, without modification.
''
chmod u+w ./.version
echo ${version} > ../../.version
'';
mesonFlags = [
];
meta = {
platforms = lib.platforms.unix ++ lib.platforms.windows;
};
})

1
src/libflake-tests/build-utils-meson
View file

@ -1 +0,0 @@
../../build-utils-meson

52
src/libflake-tests/flakeref.cc
View file

@ -7,18 +7,60 @@ namespace nix {
/* ----------- tests for flake/flakeref.hh --------------------------------------------------*/
/* ----------------------------------------------------------------------------
* to_string
* --------------------------------------------------------------------------*/
TEST(parseFlakeRef, path) {
experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes);
fetchers::Settings fetchSettings;
{
auto s = "/foo/bar";
auto flakeref = parseFlakeRef(fetchSettings, s);
ASSERT_EQ(flakeref.to_string(), "path:/foo/bar");
}
{
auto s = "/foo/bar?revCount=123&rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
auto flakeref = parseFlakeRef(fetchSettings, s);
ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&revCount=123");
}
{
auto s = "/foo/bar?xyzzy=123";
EXPECT_THROW(
parseFlakeRef(fetchSettings, s),
Error);
}
{
auto s = "/foo/bar#bla";
EXPECT_THROW(
parseFlakeRef(fetchSettings, s),
Error);
}
{
auto s = "/foo/bar#bla";
auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s);
ASSERT_EQ(flakeref.to_string(), "path:/foo/bar");
ASSERT_EQ(fragment, "bla");
}
{
auto s = "/foo/bar?revCount=123#bla";
auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s);
ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?revCount=123");
ASSERT_EQ(fragment, "bla");
}
}
TEST(to_string, doesntReencodeUrl) {
fetchers::Settings fetchSettings;
auto s = "http://localhost:8181/test/+3d.tar.gz";
auto flakeref = parseFlakeRef(fetchSettings, s);
auto parsed = flakeref.to_string();
auto unparsed = flakeref.to_string();
auto expected = "http://localhost:8181/test/%2B3d.tar.gz";
ASSERT_EQ(parsed, expected);
ASSERT_EQ(unparsed, expected);
}
}

17
src/libflake-tests/meson.build
View file

@ -4,8 +4,6 @@ project('nix-flake-tests', 'cpp',
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
@ -14,20 +12,19 @@ project('nix-flake-tests', 'cpp',
cxx = meson.get_compiler('cpp')
subdir('build-utils-meson/deps-lists')
subdir('nix-meson-build-support/deps-lists')
deps_private_maybe_subproject = [
dependency('nix-expr-test-support'),
dependency('nix-flake'),
dependency('nix-flake-c'),
]
deps_public_maybe_subproject = [
]
subdir('build-utils-meson/subprojects')
subdir('nix-meson-build-support/subprojects')
subdir('build-utils-meson/threads')
subdir('build-utils-meson/export-all-symbols')
subdir('build-utils-meson/windows-version')
subdir('nix-meson-build-support/export-all-symbols')
subdir('nix-meson-build-support/windows-version')
rapidcheck = dependency('rapidcheck')
deps_private += rapidcheck
@ -44,10 +41,11 @@ add_project_arguments(
language : 'cpp',
)
subdir('build-utils-meson/diagnostics')
subdir('nix-meson-build-support/common')
sources = files(
'flakeref.cc',
'nix_api_flake.cc',
'url-name.cc',
)
@ -70,6 +68,7 @@ test(
this_exe,
env : {
'_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data',
'NIX_CONFIG': 'extra-experimental-features = flakes',
},
protocol : 'gtest',
)

1
src/libflake-tests/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

51
src/libflake-tests/nix_api_flake.cc Normal file
View file

@ -0,0 +1,51 @@
#include "nix_api_store.h"
#include "nix_api_store_internal.h"
#include "nix_api_util.h"
#include "nix_api_util_internal.h"
#include "nix_api_expr.h"
#include "nix_api_value.h"
#include "nix_api_flake.h"
#include "tests/nix_api_expr.hh"
#include "tests/string_callback.hh"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
namespace nixC {
TEST_F(nix_api_store_test, nix_api_init_global_getFlake_exists)
{
nix_libstore_init(ctx);
assert_ctx_ok();
nix_libexpr_init(ctx);
assert_ctx_ok();
auto settings = nix_flake_settings_new(ctx);
assert_ctx_ok();
ASSERT_NE(nullptr, settings);
nix_flake_init_global(ctx, settings);
assert_ctx_ok();
nix_eval_state_builder * builder = nix_eval_state_builder_new(ctx, store);
ASSERT_NE(nullptr, builder);
assert_ctx_ok();
auto state = nix_eval_state_build(ctx, builder);
assert_ctx_ok();
ASSERT_NE(nullptr, state);
nix_eval_state_builder_free(builder);
auto value = nix_alloc_value(ctx, state);
assert_ctx_ok();
ASSERT_NE(nullptr, value);
nix_err err = nix_expr_eval_from_string(ctx, state, "builtins.getFlake", ".", value);
assert_ctx_ok();
ASSERT_EQ(NIX_OK, err);
ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(ctx, value));
}
} // namespace nixC

11
src/libflake-tests/package.nix
View file

@ -4,6 +4,7 @@
, mkMesonExecutable
, nix-flake
, nix-flake-c
, nix-expr-test-support
, rapidcheck
@ -26,8 +27,8 @@ mkMesonExecutable (finalAttrs: {
workDir = ./.;
fileset = fileset.unions [
../../build-utils-meson
./build-utils-meson
../../nix-meson-build-support
./nix-meson-build-support
../../.version
./.version
./meson.build
@ -38,6 +39,7 @@ mkMesonExecutable (finalAttrs: {
buildInputs = [
nix-flake
nix-flake-c
nix-expr-test-support
rapidcheck
gtest
@ -54,10 +56,6 @@ mkMesonExecutable (finalAttrs: {
mesonFlags = [
];
env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
passthru = {
tests = {
run = runCommand "${finalAttrs.pname}-run" {
@ -67,6 +65,7 @@ mkMesonExecutable (finalAttrs: {
mkdir -p "$HOME"
'' + ''
export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
export NIX_CONFIG="extra-experimental-features = flakes"
${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
touch $out
'');

1
src/libflake/build-utils-meson
View file

@ -1 +0,0 @@
../../build-utils-meson

294
src/libflake/flake/flake.cc
View file

@ -21,29 +21,29 @@ using namespace flake;
namespace flake {
typedef std::pair<StorePath, FlakeRef> FetchedFlake;
typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
struct FetchedFlake
{
FlakeRef lockedRef;
StorePath storePath;
};
typedef std::map<FlakeRef, FetchedFlake> FlakeCache;
static std::optional<FetchedFlake> lookupInFlakeCache(
const FlakeCache & flakeCache,
const FlakeRef & flakeRef)
{
// FIXME: inefficient.
for (auto & i : flakeCache) {
if (flakeRef == i.first) {
debug("mapping '%s' to previously seen input '%s' -> '%s",
flakeRef, i.first, i.second.second);
return i.second;
}
}
return std::nullopt;
auto i = flakeCache.find(flakeRef);
if (i == flakeCache.end()) return std::nullopt;
debug("mapping '%s' to previously seen input '%s' -> '%s",
flakeRef, i->first, i->second.lockedRef);
return i->second;
}
static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
EvalState & state,
const FlakeRef & originalRef,
bool allowLookup,
bool useRegistries,
FlakeCache & flakeCache)
{
auto fetched = lookupInFlakeCache(flakeCache, originalRef);
@ -51,32 +51,39 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
if (!fetched) {
if (originalRef.input.isDirect()) {
fetched.emplace(originalRef.fetchTree(state.store));
auto [storePath, lockedRef] = originalRef.fetchTree(state.store);
fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath});
} else {
if (allowLookup) {
resolvedRef = originalRef.resolve(state.store);
auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
flakeCache.push_back({resolvedRef, *fetchedResolved});
fetched.emplace(*fetchedResolved);
if (useRegistries) {
resolvedRef = originalRef.resolve(
state.store,
[](fetchers::Registry::RegistryType type) {
/* Only use the global registry and CLI flags
to resolve indirect flakerefs. */
return type == fetchers::Registry::Flag || type == fetchers::Registry::Global;
});
fetched = lookupInFlakeCache(flakeCache, originalRef);
if (!fetched) {
auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store);
fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath});
}
flakeCache.insert_or_assign(resolvedRef, *fetched);
}
else {
throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
}
}
flakeCache.push_back({originalRef, *fetched});
flakeCache.insert_or_assign(originalRef, *fetched);
}
auto [storePath, lockedRef] = *fetched;
debug("got tree '%s' from '%s'",
state.store->printStorePath(storePath), lockedRef);
state.store->printStorePath(fetched->storePath), fetched->lockedRef);
state.allowPath(storePath);
state.allowPath(fetched->storePath);
assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store));
assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store));
return {std::move(storePath), resolvedRef, lockedRef};
return {fetched->storePath, resolvedRef, fetched->lockedRef};
}
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
@ -95,12 +102,19 @@ static void expectType(EvalState & state, ValueType type,
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
EvalState & state, Value * value, const PosIdx pos,
const std::optional<Path> & baseDir, InputPath lockRootPath);
EvalState & state,
Value * value,
const PosIdx pos,
const InputPath & lockRootPath,
const SourcePath & flakeDir);
static FlakeInput parseFlakeInput(EvalState & state,
std::string_view inputName, Value * value, const PosIdx pos,
const std::optional<Path> & baseDir, InputPath lockRootPath)
static FlakeInput parseFlakeInput(
EvalState & state,
std::string_view inputName,
Value * value,
const PosIdx pos,
const InputPath & lockRootPath,
const SourcePath & flakeDir)
{
expectType(state, nAttrs, *value, pos);
@ -117,14 +131,25 @@ static FlakeInput parseFlakeInput(EvalState & state,
for (auto & attr : *value->attrs()) {
try {
if (attr.name == sUrl) {
expectType(state, nString, *attr.value, attr.pos);
url = attr.value->string_view();
forceTrivialValue(state, *attr.value, pos);
if (attr.value->type() == nString)
url = attr.value->string_view();
else if (attr.value->type() == nPath) {
auto path = attr.value->path();
if (path.accessor != flakeDir.accessor)
throw Error("input path '%s' at %s must be in the same source tree as %s",
path, state.positions[attr.pos], flakeDir);
url = "path:" + flakeDir.path.makeRelative(path.path);
}
else
throw Error("expected a string or a path but got %s at %s",
showType(attr.value->type()), state.positions[attr.pos]);
attrs.emplace("url", *url);
} else if (attr.name == sFlake) {
expectType(state, nBool, *attr.value, attr.pos);
input.isFlake = attr.value->boolean();
} else if (attr.name == sInputs) {
input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath);
input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath, flakeDir);
} else if (attr.name == sFollows) {
expectType(state, nString, *attr.value, attr.pos);
auto follows(parseInputPath(attr.value->c_str()));
@ -182,7 +207,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
if (!attrs.empty())
throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]);
if (url)
input.ref = parseFlakeRef(state.fetchSettings, *url, baseDir, true, input.isFlake);
input.ref = parseFlakeRef(state.fetchSettings, *url, {}, true, input.isFlake, true);
}
if (!input.follows && !input.ref)
@ -192,8 +217,11 @@ static FlakeInput parseFlakeInput(EvalState & state,
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
EvalState & state, Value * value, const PosIdx pos,
const std::optional<Path> & baseDir, InputPath lockRootPath)
EvalState & state,
Value * value,
const PosIdx pos,
const InputPath & lockRootPath,
const SourcePath & flakeDir)
{
std::map<FlakeId, FlakeInput> inputs;
@ -205,8 +233,8 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
state.symbols[inputAttr.name],
inputAttr.value,
inputAttr.pos,
baseDir,
lockRootPath));
lockRootPath,
flakeDir));
}
return inputs;
@ -220,7 +248,8 @@ static Flake readFlake(
const SourcePath & rootDir,
const InputPath & lockRootPath)
{
auto flakePath = rootDir / CanonPath(resolvedRef.subdir) / "flake.nix";
auto flakeDir = rootDir / CanonPath(resolvedRef.subdir);
auto flakePath = flakeDir / "flake.nix";
// NOTE evalFile forces vInfo to be an attrset because mustBeTrivial is true.
Value vInfo;
@ -241,7 +270,7 @@ static Flake readFlake(
auto sInputs = state.symbols.create("inputs");
if (auto inputs = vInfo.attrs()->get(sInputs))
flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME
flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath, flakeDir);
auto sOutputs = state.symbols.create("outputs");
@ -316,25 +345,20 @@ static Flake readFlake(
static Flake getFlake(
EvalState & state,
const FlakeRef & originalRef,
bool allowLookup,
bool useRegistries,
FlakeCache & flakeCache,
InputPath lockRootPath)
const InputPath & lockRootPath)
{
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, originalRef, allowLookup, flakeCache);
state, originalRef, useRegistries, flakeCache);
return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootPath);
}
Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup, FlakeCache & flakeCache)
{
return getFlake(state, originalRef, allowLookup, flakeCache, {});
}
Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistries)
{
FlakeCache flakeCache;
return getFlake(state, originalRef, allowLookup, flakeCache);
return getFlake(state, originalRef, useRegistries, flakeCache, {});
}
static LockFile readLockFile(
@ -360,7 +384,7 @@ LockedFlake lockFlake(
auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
auto flake = getFlake(state, topRef, useRegistries, flakeCache);
auto flake = getFlake(state, topRef, useRegistries, flakeCache, {});
if (lockFlags.applyNixConfig) {
flake.config.apply(settings);
@ -379,13 +403,29 @@ LockedFlake lockFlake(
debug("old lock file: %s", oldLockFile);
std::map<InputPath, FlakeInput> overrides;
struct OverrideTarget
{
FlakeInput input;
SourcePath sourcePath;
std::optional<InputPath> parentInputPath; // FIXME: rename to inputPathPrefix?
};
std::map<InputPath, OverrideTarget> overrides;
std::set<InputPath> explicitCliOverrides;
std::set<InputPath> overridesUsed, updatesUsed;
std::map<ref<Node>, SourcePath> nodePaths;
for (auto & i : lockFlags.inputOverrides) {
overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
overrides.emplace(
i.first,
OverrideTarget {
.input = FlakeInput { .ref = i.second },
/* Note: any relative overrides
(e.g. `--override-input B/C "path:./foo/bar"`)
are interpreted relative to the top-level
flake. */
.sourcePath = flake.path,
});
explicitCliOverrides.insert(i.first);
}
@ -398,8 +438,8 @@ LockedFlake lockFlake(
ref<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode,
const InputPath & lockRootPath,
const Path & parentPath,
const InputPath & followsPrefix,
const SourcePath & sourcePath,
bool trustLock)>
computeLocks;
@ -414,8 +454,13 @@ LockedFlake lockFlake(
/* The old node, if any, from which locks can be
copied. */
std::shared_ptr<const Node> oldNode,
const InputPath & lockRootPath,
const Path & parentPath,
/* The prefix relative to which 'follows' should be
interpreted. When a node is initially locked, it's
relative to the node's flake; when it's already locked,
it's relative to the root of the lock file. */
const InputPath & followsPrefix,
/* The source path of this node's flake. */
const SourcePath & sourcePath,
bool trustLock)
{
debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
@ -427,7 +472,12 @@ LockedFlake lockFlake(
auto inputPath(inputPathPrefix);
inputPath.push_back(id);
inputPath.push_back(idOverride);
overrides.insert_or_assign(inputPath, inputOverride);
overrides.emplace(inputPath,
OverrideTarget {
.input = inputOverride,
.sourcePath = sourcePath,
.parentInputPath = inputPathPrefix
});
}
}
@ -459,13 +509,18 @@ LockedFlake lockFlake(
auto i = overrides.find(inputPath);
bool hasOverride = i != overrides.end();
bool hasCliOverride = explicitCliOverrides.contains(inputPath);
if (hasOverride) {
if (hasOverride)
overridesUsed.insert(inputPath);
// Respect the “flakeness” of the input even if we
// override it
i->second.isFlake = input2.isFlake;
}
auto & input = hasOverride ? i->second : input2;
auto input = hasOverride ? i->second.input : input2;
/* Resolve relative 'path:' inputs relative to
the source path of the overrider. */
auto overridenSourcePath = hasOverride ? i->second.sourcePath : sourcePath;
/* Respect the "flakeness" of the input even if we
override it. */
if (hasOverride)
input.isFlake = input2.isFlake;
/* Resolve 'follows' later (since it may refer to an input
path we haven't processed yet. */
@ -481,6 +536,33 @@ LockedFlake lockFlake(
assert(input.ref);
auto overridenParentPath =
input.ref->input.isRelative()
? std::optional<InputPath>(hasOverride ? i->second.parentInputPath : inputPathPrefix)
: std::nullopt;
auto resolveRelativePath = [&]() -> std::optional<SourcePath>
{
if (auto relativePath = input.ref->input.isRelative()) {
return SourcePath {
overridenSourcePath.accessor,
CanonPath(*relativePath, overridenSourcePath.path.parent().value())
};
} else
return std::nullopt;
};
/* Get the input flake, resolve 'path:./...'
flakerefs relative to the parent flake. */
auto getInputFlake = [&]()
{
if (auto resolvedPath = resolveRelativePath()) {
return readFlake(state, *input.ref, *input.ref, *input.ref, *resolvedPath, inputPath);
} else {
return getFlake(state, *input.ref, useRegistries, flakeCache, inputPath);
}
};
/* Do we have an entry in the existing lock file?
And the input is not in updateInputs? */
std::shared_ptr<LockedNode> oldLock;
@ -494,6 +576,7 @@ LockedFlake lockFlake(
if (oldLock
&& oldLock->originalRef == *input.ref
&& oldLock->parentPath == overridenParentPath
&& !hasCliOverride)
{
debug("keeping existing input '%s'", inputPathS);
@ -502,7 +585,10 @@ LockedFlake lockFlake(
didn't change and there is no override from a
higher level flake. */
auto childNode = make_ref<LockedNode>(
oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
oldLock->lockedRef,
oldLock->originalRef,
oldLock->isFlake,
oldLock->parentPath);
node->inputs.insert_or_assign(id, childNode);
@ -544,7 +630,7 @@ LockedFlake lockFlake(
break;
}
}
auto absoluteFollows(lockRootPath);
auto absoluteFollows(followsPrefix);
absoluteFollows.insert(absoluteFollows.end(), follows->begin(), follows->end());
fakeInputs.emplace(i.first, FlakeInput {
.follows = absoluteFollows,
@ -554,11 +640,12 @@ LockedFlake lockFlake(
}
if (mustRefetch) {
auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath);
auto inputFlake = getInputFlake();
nodePaths.emplace(childNode, inputFlake.path.parent());
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false);
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, followsPrefix,
inputFlake.path, false);
} else {
computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true);
computeLocks(fakeInputs, childNode, inputPath, oldLock, followsPrefix, sourcePath, true);
}
} else {
@ -566,7 +653,9 @@ LockedFlake lockFlake(
this input. */
debug("creating new input '%s'", inputPathS);
if (!lockFlags.allowUnlocked && !input.ref->input.isLocked())
if (!lockFlags.allowUnlocked
&& !input.ref->input.isLocked()
&& !input.ref->input.isRelative())
throw Error("cannot update unlocked flake input '%s' in pure mode", inputPathS);
/* Note: in case of an --override-input, we use
@ -579,17 +668,13 @@ LockedFlake lockFlake(
auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref;
if (input.isFlake) {
Path localPath = parentPath;
FlakeRef localRef = *input.ref;
auto inputFlake = getInputFlake();
// If this input is a path, recurse it down.
// This allows us to resolve path inputs relative to the current flake.
if (localRef.input.getType() == "path")
localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath);
auto childNode = make_ref<LockedNode>(inputFlake.lockedRef, ref);
auto childNode = make_ref<LockedNode>(
inputFlake.lockedRef,
ref,
true,
overridenParentPath);
node->inputs.insert_or_assign(id, childNode);
@ -610,18 +695,27 @@ LockedFlake lockFlake(
oldLock
? std::dynamic_pointer_cast<const Node>(oldLock)
: readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(),
oldLock ? lockRootPath : inputPath,
localPath,
oldLock ? followsPrefix : inputPath,
inputFlake.path,
false);
}
else {
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, useRegistries, flakeCache);
auto [path, lockedRef] = [&]() -> std::tuple<SourcePath, FlakeRef>
{
// Handle non-flake 'path:./...' inputs.
if (auto resolvedPath = resolveRelativePath()) {
return {*resolvedPath, *input.ref};
} else {
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, useRegistries, flakeCache);
return {state.rootPath(state.store->toRealPath(storePath)), lockedRef};
}
}();
auto childNode = make_ref<LockedNode>(lockedRef, ref, false);
auto childNode = make_ref<LockedNode>(lockedRef, ref, false, overridenParentPath);
nodePaths.emplace(childNode, state.rootPath(state.store->toRealPath(storePath)));
nodePaths.emplace(childNode, path);
node->inputs.insert_or_assign(id, childNode);
}
@ -634,9 +728,6 @@ LockedFlake lockFlake(
}
};
// Bring in the current ref for relative path resolution if we have it
auto parentPath = flake.path.parent().path.abs();
nodePaths.emplace(newLockFile.root, flake.path.parent());
computeLocks(
@ -645,7 +736,7 @@ LockedFlake lockFlake(
{},
lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(),
{},
parentPath,
flake.path,
false);
for (auto & i : lockFlags.inputOverrides)
@ -671,7 +762,11 @@ LockedFlake lockFlake(
if (lockFlags.writeLockFile) {
if (sourcePath || lockFlags.outputLockFilePath) {
if (auto unlockedInput = newLockFile.isUnlocked()) {
if (auto unlockedInput = newLockFile.isUnlocked(state.fetchSettings)) {
if (lockFlags.failOnUnlocked)
throw Error(
"Will not write lock file of flake '%s' because it has an unlocked input ('%s'). "
"Use '--allow-dirty-locks' to allow this anyway.", topRef, *unlockedInput);
if (state.fetchSettings.warnDirty)
warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput);
} else {
@ -686,7 +781,7 @@ LockedFlake lockFlake(
writeFile(*lockFlags.outputLockFilePath, newLockFileS);
} else {
auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
auto outputLockFilePath = *sourcePath + "/" + relPath;
auto outputLockFilePath = *sourcePath / relPath;
bool lockFileExists = pathExists(outputLockFilePath);
@ -724,8 +819,7 @@ LockedFlake lockFlake(
repo, so we should re-read it. FIXME: we could
also just clear the 'rev' field... */
auto prevLockedRef = flake.lockedRef;
FlakeCache dummyCache;
flake = getFlake(state, topRef, useRegistries, dummyCache);
flake = getFlake(state, topRef, useRegistries);
if (lockFlags.commitLockFile &&
flake.lockedRef.input.getRev() &&
@ -816,7 +910,7 @@ void callFlake(EvalState & state,
assert(vFetchFinalTree);
Value * args[] = {vLocks, &vOverrides, *vFetchFinalTree};
state.callFunction(*vCallFlake, 3, args, vRes, noPos);
state.callFunction(*vCallFlake, args, vRes, noPos);
}
void initLib(const Settings & settings)
@ -970,9 +1064,11 @@ static RegisterPrimOp r4({
}
std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store) const
std::optional<Fingerprint> LockedFlake::getFingerprint(
ref<Store> store,
const fetchers::Settings & fetchSettings) const
{
if (lockFile.isUnlocked()) return std::nullopt;
if (lockFile.isUnlocked(fetchSettings)) return std::nullopt;
auto fingerprint = flake.lockedRef.input.getFingerprint(store);
if (!fingerprint) return std::nullopt;

11
src/libflake/flake/flake.hh
View file

@ -110,7 +110,7 @@ struct Flake
}
};
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool useRegistries);
/**
* Fingerprint of a locked flake; used as a cache key.
@ -129,7 +129,9 @@ struct LockedFlake
*/
std::map<ref<Node>, SourcePath> nodePaths;
std::optional<Fingerprint> getFingerprint(ref<Store> store) const;
std::optional<Fingerprint> getFingerprint(
ref<Store> store,
const fetchers::Settings & fetchSettings) const;
};
struct LockFlags
@ -156,6 +158,11 @@ struct LockFlags
*/
bool writeLockFile = true;
/**
* Throw an exception when the flake has an unlocked input.
*/
bool failOnUnlocked = false;
/**
* Whether to use the registries to lookup indirect flake
* references like 'nixpkgs'.

124
src/libflake/flake/flakeref.cc
View file

@ -3,7 +3,6 @@
#include "url.hh"
#include "url-parts.hh"
#include "fetchers.hh"
#include "registry.hh"
namespace nix {
@ -36,7 +35,9 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
return str;
}
FlakeRef FlakeRef::resolve(ref<Store> store) const
FlakeRef FlakeRef::resolve(
ref<Store> store,
const fetchers::RegistryFilter & filter) const
{
auto [input2, extraAttrs] = lookupInRegistries(store, input);
return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
@ -47,9 +48,10 @@ FlakeRef parseFlakeRef(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake)
bool isFlake,
bool preserveRelativePaths)
{
auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake);
auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, preserveRelativePaths);
if (fragment != "")
throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
return flakeRef;
@ -67,35 +69,43 @@ std::optional<FlakeRef> maybeParseFlakeRef(
}
}
static std::pair<FlakeRef, std::string> fromParsedURL(
const fetchers::Settings & fetchSettings,
ParsedURL && parsedURL,
bool isFlake)
{
auto dir = getOr(parsedURL.query, "dir", "");
parsedURL.query.erase("dir");
std::string fragment;
std::swap(fragment, parsedURL.fragment);
return {FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment};
}
std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
const fetchers::Settings & fetchSettings,
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake)
bool isFlake,
bool preserveRelativePaths)
{
std::string path = url;
std::string fragment = "";
std::map<std::string, std::string> query;
auto pathEnd = url.find_first_of("#?");
auto fragmentStart = pathEnd;
if (pathEnd != std::string::npos && url[pathEnd] == '?') {
fragmentStart = url.find("#");
}
if (pathEnd != std::string::npos) {
path = url.substr(0, pathEnd);
}
if (fragmentStart != std::string::npos) {
fragment = percentDecode(url.substr(fragmentStart+1));
}
if (pathEnd != std::string::npos && fragmentStart != std::string::npos && url[pathEnd] == '?') {
query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1));
}
static std::regex pathFlakeRegex(
R"(([^?#]*)(\?([^#]*))?(#(.*))?)",
std::regex::ECMAScript);
std::smatch match;
auto succeeds = std::regex_match(url, match, pathFlakeRegex);
assert(succeeds);
auto path = match[1].str();
auto query = decodeQuery(match[3]);
auto fragment = percentDecode(match[5].str());
if (baseDir) {
/* Check if 'url' is a path (either absolute or relative
to 'baseDir'). If so, search upward to the root of the
repo (i.e. the directory containing .git). */
to 'baseDir'). If so, search upward to the root of the
repo (i.e. the directory containing .git). */
path = absPath(path, baseDir);
@ -144,15 +154,12 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
while (flakeRoot != "/") {
if (pathExists(flakeRoot + "/.git")) {
auto base = std::string("git+file://") + flakeRoot;
auto parsedURL = ParsedURL{
.url = base, // FIXME
.base = base,
.scheme = "git+file",
.authority = "",
.path = flakeRoot,
.query = query,
.fragment = fragment,
};
if (subdir != "") {
@ -164,9 +171,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
if (pathExists(flakeRoot + "/.git/shallow"))
parsedURL.query.insert_or_assign("shallow", "1");
return std::make_pair(
FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")),
fragment);
return fromParsedURL(fetchSettings, std::move(parsedURL), isFlake);
}
subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
@ -175,26 +180,27 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
}
} else {
if (!hasPrefix(path, "/"))
if (!preserveRelativePaths && !isAbsolute(path))
throw BadURL("flake reference '%s' is not an absolute path", url);
path = canonPath(path + "/" + getOr(query, "dir", ""));
}
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "path");
attrs.insert_or_assign("path", path);
return fromParsedURL(fetchSettings, {
.scheme = "path",
.authority = "",
.path = path,
.query = query,
.fragment = fragment
}, isFlake);
}
return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment);
};
/* Check if 'url' is a flake ID. This is an abbreviated syntax for
'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
/**
* Check if `url` is a flake ID. This is an abbreviated syntax for
* `flake:<flake-id>?ref=<ref>&rev=<rev>`.
*/
static std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
const fetchers::Settings & fetchSettings,
const std::string & url,
bool isFlake
)
bool isFlake)
{
std::smatch match;
@ -205,8 +211,6 @@ static std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
if (std::regex_match(url, match, flakeRegex)) {
auto parsedURL = ParsedURL{
.url = url,
.base = "flake:" + match.str(1),
.scheme = "flake",
.authority = "",
.path = match[1],
@ -224,25 +228,18 @@ std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
const fetchers::Settings & fetchSettings,
const std::string & url,
const std::optional<Path> & baseDir,
bool isFlake
)
bool isFlake)
{
ParsedURL parsedURL;
try {
parsedURL = parseURL(url);
auto parsed = parseURL(url);
if (baseDir
&& (parsed.scheme == "path" || parsed.scheme == "git+file")
&& !isAbsolute(parsed.path))
parsed.path = absPath(parsed.path, *baseDir);
return fromParsedURL(fetchSettings, std::move(parsed), isFlake);
} catch (BadURL &) {
return std::nullopt;
}
std::string fragment;
std::swap(fragment, parsedURL.fragment);
auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake);
input.parent = baseDir;
return std::make_pair(
FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
fragment);
}
std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
@ -250,18 +247,17 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake)
bool isFlake,
bool preserveRelativePaths)
{
using namespace fetchers;
std::smatch match;
if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) {
return *res;
} else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) {
return *res;
} else {
return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake);
return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, preserveRelativePaths);
}
}

16
src/libflake/flake/flakeref.hh
View file

@ -6,6 +6,7 @@
#include "types.hh"
#include "fetchers.hh"
#include "outputs-spec.hh"
#include "registry.hh"
namespace nix {
@ -48,6 +49,11 @@ struct FlakeRef
bool operator ==(const FlakeRef & other) const = default;
bool operator <(const FlakeRef & other) const
{
return std::tie(input, subdir) < std::tie(other.input, other.subdir);
}
FlakeRef(fetchers::Input && input, const Path & subdir)
: input(std::move(input)), subdir(subdir)
{ }
@ -57,7 +63,9 @@ struct FlakeRef
fetchers::Attrs toAttrs() const;
FlakeRef resolve(ref<Store> store) const;
FlakeRef resolve(
ref<Store> store,
const fetchers::RegistryFilter & filter = {}) const;
static FlakeRef fromAttrs(
const fetchers::Settings & fetchSettings,
@ -76,7 +84,8 @@ FlakeRef parseFlakeRef(
const std::string & url,
const std::optional<Path> & baseDir = {},
bool allowMissing = false,
bool isFlake = true);
bool isFlake = true,
bool preserveRelativePaths = false);
/**
* @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
@ -94,7 +103,8 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
const std::string & url,
const std::optional<Path> & baseDir = {},
bool allowMissing = false,
bool isFlake = true);
bool isFlake = true,
bool preserveRelativePaths = false);
/**
* @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)

17
src/libflake/flake/lockfile.cc
View file

@ -10,6 +10,7 @@
#include <nlohmann/json.hpp>
#include "strings.hh"
#include "flake/settings.hh"
namespace nix::flake {
@ -42,9 +43,10 @@ LockedNode::LockedNode(
: lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info"
, originalRef(getFlakeRef(fetchSettings, json, "original", nullptr))
, isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
, parentPath(json.find("parent") != json.end() ? (std::optional<InputPath>) json["parent"] : std::nullopt)
{
if (!lockedRef.input.isLocked())
throw Error("lock file contains unlocked input '%s'",
if (!lockedRef.input.isConsideredLocked(fetchSettings) && !lockedRef.input.isRelative())
throw Error("Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.",
fetchers::attrsToJSON(lockedRef.input.toAttrs()));
// For backward compatibility, lock file entries are implicitly final.
@ -197,10 +199,12 @@ std::pair<nlohmann::json, LockFile::KeyMap> LockFile::toJSON() const
/* For backward compatibility, omit the "__final"
attribute. We never allow non-final inputs in lock files
anyway. */
assert(lockedNode->lockedRef.input.isFinal());
assert(lockedNode->lockedRef.input.isFinal() || lockedNode->lockedRef.input.isRelative());
n["locked"].erase("__final");
if (!lockedNode->isFlake)
n["flake"] = false;
if (lockedNode->parentPath)
n["parent"] = *lockedNode->parentPath;
}
nodes[key] = std::move(n);
@ -228,7 +232,7 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
return stream;
}
std::optional<FlakeRef> LockFile::isUnlocked() const
std::optional<FlakeRef> LockFile::isUnlocked(const fetchers::Settings & fetchSettings) const
{
std::set<ref<const Node>> nodes;
@ -247,7 +251,10 @@ std::optional<FlakeRef> LockFile::isUnlocked() const
for (auto & i : nodes) {
if (i == ref<const Node>(root)) continue;
auto node = i.dynamic_pointer_cast<const LockedNode>();
if (node && (!node->lockedRef.input.isLocked() || !node->lockedRef.input.isFinal()))
if (node
&& (!node->lockedRef.input.isConsideredLocked(fetchSettings)
|| !node->lockedRef.input.isFinal())
&& !node->lockedRef.input.isRelative())
return node->lockedRef;
}

14
src/libflake/flake/lockfile.hh
View file

@ -38,11 +38,19 @@ struct LockedNode : Node
FlakeRef lockedRef, originalRef;
bool isFlake = true;
/* The node relative to which relative source paths
(e.g. 'path:../foo') are interpreted. */
std::optional<InputPath> parentPath;
LockedNode(
const FlakeRef & lockedRef,
const FlakeRef & originalRef,
bool isFlake = true)
: lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake)
bool isFlake = true,
std::optional<InputPath> parentPath = {})
: lockedRef(lockedRef)
, originalRef(originalRef)
, isFlake(isFlake)
, parentPath(parentPath)
{ }
LockedNode(
@ -71,7 +79,7 @@ struct LockFile
* Check whether this lock file has any unlocked or non-final
* inputs. If so, return one.
*/
std::optional<FlakeRef> isUnlocked() const;
std::optional<FlakeRef> isUnlocked(const fetchers::Settings & fetchSettings) const;
bool operator ==(const LockFile & other) const;

2
src/libflake/flake/settings.hh
View file

@ -29,7 +29,7 @@ struct Settings : public Config
this,
false,
"accept-flake-config",
"Whether to accept nix configuration from a flake without prompting.",
"Whether to accept Nix configuration settings from a flake without prompting.",
{},
true,
Xp::Flakes};

Some files were not shown because too many files have changed in this diff Show more