nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-12 05:26:02 +01:00
Code Activity

Merge remote-tracking branch 'origin/master' into relative-flakes

Browse source
This commit is contained in:
Eelco Dolstra 2024-09-16 13:45:58 +02:00
commit 21fc07c1a4
1142 changed files with 31230 additions and 23784 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/libexpr/.version Symbolic link
View file

@ -0,0 +1 @@
../../.version

4
src/libexpr/attr-path.cc
View file

@ -76,7 +76,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
if (!a) {
std::set<std::string> attrNames;
for (auto & attr : *v->attrs())
attrNames.insert(state.symbols[attr.name]);
attrNames.insert(std::string(state.symbols[attr.name]));
auto suggestions = Suggestions::bestMatches(attrNames, attr);
throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);
@ -134,7 +134,7 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
} catch (std::invalid_argument & e) {
fail();
abort();
unreachable();
}
}

5
src/libexpr/attr-set.hh
View file

@ -5,7 +5,6 @@
#include "symbol-table.hh"
#include <algorithm>
#include <optional>
namespace nix {
@ -28,9 +27,9 @@ struct Attr
Attr(Symbol name, Value * value, PosIdx pos = noPos)
: name(name), pos(pos), value(value) { };
Attr() { };
bool operator < (const Attr & a) const
auto operator <=> (const Attr & a) const
{
return name < a.name;
return name <=> a.name;
}
};

1
src/libexpr/build-utils-meson Symbolic link
View file

@ -0,0 +1 @@
../../build-utils-meson

0
src/libexpr/flake/call-flake.nix → src/libexpr/call-flake.nix
View file

59
src/libexpr/eval-cache.cc
View file

@ -4,9 +4,30 @@
#include "eval.hh"
#include "eval-inline.hh"
#include "store-api.hh"
// Need specialization involving `SymbolStr` just in this one module.
#include "strings-inline.hh"
namespace nix::eval_cache {
CachedEvalError::CachedEvalError(ref<AttrCursor> cursor, Symbol attr)
: EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr))
, cursor(cursor), attr(attr)
{ }
void CachedEvalError::force()
{
auto & v = cursor->forceValue();
if (v.type() == nAttrs) {
auto a = v.attrs()->get(this->attr);
state.forceValue(*a->value, a->pos);
}
// Shouldn't happen.
throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr));
}
static const char * schema = R"sql(
create table if not exists Attributes (
parent integer not null,
@ -48,7 +69,7 @@ struct AttrDb
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v5";
Path cacheDir = getCacheDir() + "/eval-cache-v5";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite";
@ -76,7 +97,7 @@ struct AttrDb
{
try {
auto state(_state->lock());
if (!failed)
if (!failed && state->txn->active)
state->txn->commit();
state->txn.reset();
} catch (...) {
@ -206,7 +227,7 @@ struct AttrDb
(key.first)
(symbols[key.second])
(AttrType::ListOfStrings)
(concatStringsSep("\t", l)).exec();
(dropEmptyInitThenConcatStringsSep("\t", l)).exec();
return state->db.getLastInsertedRowId();
});
@ -307,7 +328,7 @@ struct AttrDb
case AttrType::Bool:
return {{rowId, queryAttribute.getInt(2) != 0}};
case AttrType::Int:
return {{rowId, int_t{queryAttribute.getInt(2)}}};
return {{rowId, int_t{NixInt{queryAttribute.getInt(2)}}}};
case AttrType::ListOfStrings:
return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
case AttrType::Missing:
@ -416,12 +437,12 @@ std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
std::string AttrCursor::getAttrPathStr() const
{
return concatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
}
std::string AttrCursor::getAttrPathStr(Symbol name) const
{
return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
}
Value & AttrCursor::forceValue()
@ -450,7 +471,7 @@ Value & AttrCursor::forceValue()
else if (v.type() == nBool)
cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()};
else if (v.type() == nInt)
cachedValue = {root->db->setInt(getKey(), v.integer()), int_t{v.integer()}};
cachedValue = {root->db->setInt(getKey(), v.integer().value), int_t{v.integer()}};
else if (v.type() == nAttrs)
; // FIXME: do something?
else
@ -465,12 +486,12 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
auto attrNames = getAttrs();
std::set<std::string> strAttrNames;
for (auto & name : attrNames)
strAttrNames.insert(root->state.symbols[name]);
strAttrNames.insert(std::string(root->state.symbols[name]));
return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
{
if (root->db) {
if (!cachedValue)
@ -487,12 +508,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
if (attr) {
if (std::get_if<missing_t>(&attr->second))
return nullptr;
else if (std::get_if<failed_t>(&attr->second)) {
if (forceErrors)
debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
else
throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name));
} else
else if (std::get_if<failed_t>(&attr->second))
throw CachedEvalError(ref(shared_from_this()), name);
else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
@ -537,9 +555,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
return maybeGetAttr(root->state.symbols.create(name));
}
ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
ref<AttrCursor> AttrCursor::getAttr(Symbol name)
{
auto p = maybeGetAttr(name, forceErrors);
auto p = maybeGetAttr(name);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return ref(p);
@ -550,11 +568,11 @@ ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
return getAttr(root->state.symbols.create(name));
}
OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto res = shared_from_this();
for (auto & attr : attrPath) {
auto child = res->maybeGetAttr(attr, force);
auto child = res->maybeGetAttr(attr);
if (!child) {
auto suggestions = res->getSuggestionsForAttr(attr);
return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
@ -751,8 +769,9 @@ bool AttrCursor::isDerivation()
StorePath AttrCursor::forceDerivation()
{
auto aDrvPath = getAttr(root->state.sDrvPath, true);
auto aDrvPath = getAttr(root->state.sDrvPath);
auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
drvPath.requireDerivation();
if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
/* The eval cache contains 'drvPath', but the actual path has
been garbage-collected. So force it to be regenerated. */

25
src/libexpr/eval-cache.hh
View file

@ -10,14 +10,28 @@
namespace nix::eval_cache {
MakeError(CachedEvalError, EvalError);
struct AttrDb;
class AttrCursor;
struct CachedEvalError : EvalError
{
const ref<AttrCursor> cursor;
const Symbol attr;
CachedEvalError(ref<AttrCursor> cursor, Symbol attr);
/**
* Evaluate this attribute, which should result in a regular
* `EvalError` exception being thrown.
*/
[[noreturn]]
void force();
};
class EvalCache : public std::enable_shared_from_this<EvalCache>
{
friend class AttrCursor;
friend struct CachedEvalError;
std::shared_ptr<AttrDb> db;
EvalState & state;
@ -73,6 +87,7 @@ typedef std::variant<
class AttrCursor : public std::enable_shared_from_this<AttrCursor>
{
friend class EvalCache;
friend struct CachedEvalError;
ref<EvalCache> root;
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
@ -102,11 +117,11 @@ public:
Suggestions getSuggestionsForAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
ref<AttrCursor> getAttr(Symbol name);
ref<AttrCursor> getAttr(std::string_view name);
@ -114,7 +129,7 @@ public:
* Get an attribute along a chain of attrsets. Note that this does
* not auto-call functors or functions.
*/
OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath);
std::string getString();

27
src/libexpr/eval-error.cc
View file

@ -27,8 +27,7 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(Value & value, PosIdx fallback)
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
{
error.err.traces.push_front(
Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))});
error.addTrace(error.state.positions[pos], text);
return *this;
}
@ -71,15 +70,17 @@ EvalErrorBuilder<T>::addTrace(PosIdx pos, std::string_view formatString, const A
return *this;
}
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::setIsFromExpr()
{
error.err.isFromExpr = true;
return *this;
}
template<class T>
void EvalErrorBuilder<T>::debugThrow()
{
if (error.state.debugRepl && !error.state.debugTraces.empty()) {
const DebugTrace & last = error.state.debugTraces.front();
const Env * env = &last.env;
const Expr * expr = &last.expr;
error.state.runDebugRepl(&error, *env, *expr);
}
error.state.runDebugRepl(&error);
// `EvalState` is the only class that can construct an `EvalErrorBuilder`,
// and it does so in dynamic storage. This is the final method called on
@ -91,6 +92,15 @@ void EvalErrorBuilder<T>::debugThrow()
throw error;
}
template<class T>
void EvalErrorBuilder<T>::panic()
{
logError(error.info());
printError("This is a bug! An unexpected condition occurred, causing the Nix evaluator to have to stop. If you could share a reproducible example or a core dump, please open an issue at https://github.com/NixOS/nix/issues");
abort();
}
template class EvalErrorBuilder<EvalBaseError>;
template class EvalErrorBuilder<EvalError>;
template class EvalErrorBuilder<AssertionError>;
template class EvalErrorBuilder<ThrownError>;
@ -99,7 +109,6 @@ template class EvalErrorBuilder<TypeError>;
template class EvalErrorBuilder<UndefinedVarError>;
template class EvalErrorBuilder<MissingArgumentError>;
template class EvalErrorBuilder<InfiniteRecursionError>;
template class EvalErrorBuilder<CachedEvalError>;
template class EvalErrorBuilder<InvalidPathError>;
}

29
src/libexpr/eval-error.hh
View file

@ -1,7 +1,5 @@
#pragma once
#include <algorithm>
#include "error.hh"
#include "pos-idx.hh"
@ -15,27 +13,39 @@ class EvalState;
template<class T>
class EvalErrorBuilder;
class EvalError : public Error
/**
* Base class for all errors that occur during evaluation.
*
* Most subclasses should inherit from `EvalError` instead of this class.
*/
class EvalBaseError : public Error
{
template<class T>
friend class EvalErrorBuilder;
public:
EvalState & state;
EvalError(EvalState & state, ErrorInfo && errorInfo)
EvalBaseError(EvalState & state, ErrorInfo && errorInfo)
: Error(errorInfo)
, state(state)
{
}
template<typename... Args>
explicit EvalError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
explicit EvalBaseError(EvalState & state, const std::string & formatString, const Args &... formatArgs)
: Error(formatString, formatArgs...)
, state(state)
{
}
};
/**
* `EvalError` is the base class for almost all errors that occur during evaluation.
*
* All instances of `EvalError` should show a degree of purity that allows them to be
* cached in pure mode. This means that they should not depend on the configuration or the overall environment.
*/
MakeError(EvalError, EvalBaseError);
MakeError(ParseError, Error);
MakeError(AssertionError, EvalError);
MakeError(ThrownError, AssertionError);
@ -43,7 +53,6 @@ MakeError(Abort, EvalError);
MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, EvalError);
MakeError(MissingArgumentError, EvalError);
MakeError(CachedEvalError, EvalError);
MakeError(InfiniteRecursionError, EvalError);
struct InvalidPathError : public EvalError
@ -91,6 +100,8 @@ public:
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint);
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & setIsFromExpr();
template<typename... Args>
[[nodiscard, gnu::noinline]] EvalErrorBuilder<T> &
addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs);
@ -99,6 +110,12 @@ public:
* Delete the `EvalErrorBuilder` and throw the underlying exception.
*/
[[gnu::noinline, gnu::noreturn]] void debugThrow();
/**
* A programming error or fatal condition occurred. Abort the process for core dump and debugging.
* This does not print a proper backtrace, because unwinding the stack is destructive.
*/
[[gnu::noinline, gnu::noreturn]] void panic();
};
}

117
src/libexpr/eval-gc.cc Normal file
View file

@ -0,0 +1,117 @@
#include "error.hh"
#include "environment-variables.hh"
#include "eval-settings.hh"
#include "config-global.hh"
#include "serialise.hh"
#include "eval-gc.hh"
#if HAVE_BOEHMGC
# include <pthread.h>
# if __FreeBSD__
# include <pthread_np.h>
# endif
# include <gc/gc.h>
# include <gc/gc_cpp.h>
# include <gc/gc_allocator.h>
# include <boost/coroutine2/coroutine.hpp>
# include <boost/coroutine2/protected_fixedsize_stack.hpp>
# include <boost/context/stack_context.hpp>
#endif
namespace nix {
#if HAVE_BOEHMGC
/* Called when the Boehm GC runs out of memory. */
static void * oomHandler(size_t requested)
{
/* Convert this to a proper C++ exception. */
throw std::bad_alloc();
}
static inline void initGCReal()
{
/* Initialise the Boehm garbage collector. */
/* Don't look for interior pointers. This reduces the odds of
misdetection a bit. */
GC_set_all_interior_pointers(0);
/* We don't have any roots in data segments, so don't scan from
there. */
GC_set_no_dls(1);
/* Enable perf measurements. This is just a setting; not much of a
start of something. */
GC_start_performance_measurement();
GC_INIT();
GC_set_oom_fn(oomHandler);
/* Set the initial heap size to something fairly big (25% of
physical RAM, up to a maximum of 384 MiB) so that in most cases
we don't need to garbage collect at all. (Collection has a
fairly significant overhead.) The heap size can be overridden
through libgc's GC_INITIAL_HEAP_SIZE environment variable. We
should probably also provide a nix.conf setting for this. Note
that GC_expand_hp() causes a lot of virtual, but not physical
(resident) memory to be allocated. This might be a problem on
systems that don't overcommit. */
if (!getEnv("GC_INITIAL_HEAP_SIZE")) {
size_t size = 32 * 1024 * 1024;
# if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
size_t maxSize = 384 * 1024 * 1024;
long pageSize = sysconf(_SC_PAGESIZE);
long pages = sysconf(_SC_PHYS_PAGES);
if (pageSize != -1)
size = (pageSize * pages) / 4; // 25% of RAM
if (size > maxSize)
size = maxSize;
# endif
debug("setting initial heap size to %1% bytes", size);
GC_expand_hp(size);
}
}
static size_t gcCyclesAfterInit = 0;
size_t getGCCycles()
{
assertGCInitialized();
return static_cast<size_t>(GC_get_gc_no()) - gcCyclesAfterInit;
}
#endif
static bool gcInitialised = false;
void initGC()
{
if (gcInitialised)
return;
#if HAVE_BOEHMGC
initGCReal();
gcCyclesAfterInit = GC_get_gc_no();
#endif
// NIX_PATH must override the regular setting
// See the comment in applyConfig
if (auto nixPathEnv = getEnv("NIX_PATH")) {
globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value())));
}
gcInitialised = true;
}
void assertGCInitialized()
{
assert(gcInitialised);
}
} // namespace nix

25
src/libexpr/eval-gc.hh Normal file
View file

@ -0,0 +1,25 @@
#pragma once
///@file
#include <cstddef>
namespace nix {
/**
* Initialise the Boehm GC, if applicable.
*/
void initGC();
/**
* Make sure `initGC` has already been called.
*/
void assertGCInitialized();
#ifdef HAVE_BOEHMGC
/**
* The number of GC cycles since initGC().
*/
size_t getGCCycles();
#endif
} // namespace nix

8
src/libexpr/eval-inline.hh
View file

@ -4,6 +4,7 @@
#include "print.hh"
#include "eval.hh"
#include "eval-error.hh"
#include "eval-settings.hh"
namespace nix {
@ -138,5 +139,12 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e
}
}
[[gnu::always_inline]]
inline CallDepth EvalState::addCallDepth(const PosIdx pos) {
if (callDepth > settings.maxCallDepth)
error<EvalError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
return CallDepth(callDepth);
};
}

27
src/libexpr/eval-settings.cc
View file

@ -8,7 +8,7 @@ namespace nix {
/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
static Strings parseNixPath(const std::string & s)
Strings EvalSettings::parseNixPath(const std::string & s)
{
Strings res;
@ -44,10 +44,13 @@ static Strings parseNixPath(const std::string & s)
return res;
}
EvalSettings::EvalSettings()
EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks)
: readOnlyMode{readOnlyMode}
, lookupPathHooks{lookupPathHooks}
{
auto var = getEnv("NIX_PATH");
if (var) nixPath = parseNixPath(*var);
auto var = getEnv("NIX_ABORT_ON_WARN");
if (var && (var == "1" || var == "yes" || var == "true"))
builtinsAbortOnWarn = true;
}
Strings EvalSettings::getDefaultNixPath()
@ -63,11 +66,9 @@ Strings EvalSettings::getDefaultNixPath()
}
};
if (!evalSettings.restrictEval && !evalSettings.pureEval) {
add(getNixDefExpr() + "/channels");
add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
add(rootChannelsDir());
}
add(getNixDefExpr() + "/channels");
add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
add(rootChannelsDir());
return res;
}
@ -89,20 +90,16 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url)
return std::string(url);
}
const std::string & EvalSettings::getCurrentSystem()
const std::string & EvalSettings::getCurrentSystem() const
{
const auto & evalSystem = currentSystem.get();
return evalSystem != "" ? evalSystem : settings.thisSystem.get();
}
EvalSettings evalSettings;
static GlobalConfig::Register rEvalSettings(&evalSettings);
Path getNixDefExpr()
{
return settings.useXDGBaseDirectories
? getStateDir() + "/nix/defexpr"
? getStateDir() + "/defexpr"
: getHome() + "/.nix-defexpr";
}

150
src/libexpr/eval-settings.hh
View file

@ -2,49 +2,117 @@
///@file
#include "config.hh"
#include "ref.hh"
namespace nix {
class Store;
struct EvalSettings : Config
{
EvalSettings();
/**
* Function used to interpet look path entries of a given scheme.
*
* The argument is the non-scheme part of the lookup path entry (see
* `LookupPathHooks` below).
*
* The return value is (a) whether the entry was valid, and, if so,
* what does it map to.
*
* @todo Return (`std::optional` of) `SourceAccssor` or something
* more structured instead of mere `std::string`?
*/
using LookupPathHook = std::optional<std::string>(ref<Store> store, std::string_view);
/**
* Map from "scheme" to a `LookupPathHook`.
*
* Given a lookup path value (i.e. either the whole thing, or after
* the `<key>=`) in the form of:
*
* ```
* <scheme>:<arbitrary string>
* ```
*
* if `<scheme>` is a key in this map, then `<arbitrary string>` is
* passed to the hook that is the value in this map.
*/
using LookupPathHooks = std::map<std::string, std::function<LookupPathHook>>;
EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {});
bool & readOnlyMode;
static Strings getDefaultNixPath();
static bool isPseudoUrl(std::string_view s);
static Strings parseNixPath(const std::string & s);
static std::string resolvePseudoUrl(std::string_view url);
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."};
LookupPathHooks lookupPathHooks;
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"(
Enable built-in functions that allow executing native code.
In particular, this adds:
- `builtins.importNative` *path* *symbol*
Opens dynamic shared object (DSO) at *path*, loads the function with the symbol name *symbol* from it and runs it.
The loaded function must have the following signature:
```cpp
extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v);
```
The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals.
- `builtins.exec` *arguments*
Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression.
)"};
Setting<Strings> nixPath{
this, getDefaultNixPath(), "nix-path",
this, {}, "nix-path",
R"(
List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution.
This setting determines the value of
[`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtin-constants.md#builtins-findFile).
[`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile).
The default value is
- The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH)
environment variable.
- `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`.
- Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`.
```
$HOME/.nix-defexpr/channels
nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs
$NIX_STATE_DIR/profiles/per-user/root/channels
```
If the respective paths are accessible, the default values are:
It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I).
- `$HOME/.nix-defexpr/channels`
The [user channel link](@docroot@/command-ref/files/default-nix-expression.md#user-channel-link), pointing to the current state of [channels](@docroot@/command-ref/files/channels.md) for the current user.
- `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs`
The current state of the `nixpkgs` channel for the `root` user.
- `$NIX_STATE_DIR/profiles/per-user/root/channels`
The current state of all channels for the `root` user.
These files are set up by the [Nix installer](@docroot@/installation/installing-binary.md).
See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details on the environment variable.
> **Note**
>
> If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`.
> If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty.
>
> If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`.
)", {}, false};
Setting<std::string> currentSystem{
this, "", "eval-system",
R"(
This option defines
[`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
[`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem)
in the Nix language if it is set as a non-empty string.
Otherwise, if it is defined as the empty string (the default), the value of the
[`system` ](#conf-system)
@ -58,14 +126,14 @@ struct EvalSettings : Config
* Implements the `eval-system` vs `system` defaulting logic
* described for `eval-system`.
*/
const std::string & getCurrentSystem();
const std::string & getCurrentSystem() const;
Setting<bool> restrictEval{
this, false, "restrict-eval",
R"(
If set to `true`, the Nix evaluator will not allow access to any
files outside of
[`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath),
[`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath),
or to URIs outside of
[`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris).
)"};
@ -76,10 +144,10 @@ struct EvalSettings : Config
- Restrict file system and network access to files specified by cryptographic hash
- Disable impure constants:
- [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
- [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
- [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath)
- [`builtins.storePath`](@docroot@/language/builtin-constants.md#builtins-storePath)
- [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem)
- [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime)
- [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath)
- [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath)
)"
};
@ -126,7 +194,11 @@ struct EvalSettings : Config
)"};
Setting<bool> useEvalCache{this, true, "eval-cache",
"Whether to use the flake evaluation cache."};
R"(
Whether to use the flake evaluation cache.
Certain commands won't have to evaluate when invoked for the second time with a particular version of a flake.
Intermediate results are not cached.
)"};
Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
R"(
@ -142,16 +214,40 @@ struct EvalSettings : Config
Setting<bool> builtinsTraceDebugger{this, false, "debugger-on-trace",
R"(
If set to true and the `--debugger` flag is given,
[`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) will
enter the debugger like
[`builtins.break`](@docroot@/language/builtins.md#builtins-break).
If set to true and the `--debugger` flag is given, the following functions
will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
* [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace)
* [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose)
if [`trace-verbose`](#conf-trace-verbose) is set to true.
* [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
This is useful for debugging warnings in third-party Nix code.
)"};
};
extern EvalSettings evalSettings;
Setting<bool> builtinsDebuggerOnWarn{this, false, "debugger-on-warn",
R"(
If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn)
will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
This is useful for debugging warnings in third-party Nix code.
Use [`debugger-on-trace`](#conf-debugger-on-trace) to also enter the debugger on legacy warnings that are logged with [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace).
)"};
Setting<bool> builtinsAbortOnWarn{this, false, "abort-on-warn",
R"(
If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) will throw an error when logging a warning.
This will give you a stack trace that leads to the location of the warning.
This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn).
Currently, a stack trace can only be produced when the debugger is enabled, or when evaluation is aborted.
This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
)"};
};
/**
* Conventionally part of the default nix path in impure mode.

695
src/libexpr/eval.cc
View file

File diff suppressed because it is too large Load diff

96
src/libexpr/eval.hh
View file

@ -9,14 +9,15 @@
#include "symbol-table.hh"
#include "config.hh"
#include "experimental-features.hh"
#include "position.hh"
#include "pos-table.hh"
#include "source-accessor.hh"
#include "search-path.hh"
#include "repl-exit-status.hh"
#include "ref.hh"
#include <map>
#include <optional>
#include <unordered_map>
#include <mutex>
#include <functional>
namespace nix {
@ -29,6 +30,8 @@ namespace nix {
constexpr size_t maxPrimOpArity = 8;
class Store;
namespace fetchers { struct Settings; }
struct EvalSettings;
class EvalState;
class StorePath;
struct SingleDerivedPath;
@ -38,11 +41,25 @@ namespace eval_cache {
class EvalCache;
}
/**
* Increments a count on construction and decrements on destruction.
*/
class CallDepth {
size_t & count;
public:
CallDepth(size_t & count) : count(count) {
++count;
}
~CallDepth() {
--count;
}
};
/**
* Function that implements a primop.
*/
typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v);
/**
* Info about a primitive operation, and its implementation
@ -83,7 +100,7 @@ struct PrimOp
/**
* Implementation of the primop.
*/
std::function<std::remove_pointer<PrimOpFun>::type> fun;
std::function<PrimOpFun> fun;
/**
* Optional experimental for this to be gated on.
@ -128,6 +145,8 @@ struct Constant
typedef std::map<std::string, Value *> ValMap;
#endif
typedef std::unordered_map<PosIdx, DocComment> DocCommentMap;
struct Env
{
Env * up;
@ -146,12 +165,6 @@ std::string printValue(EvalState & state, Value & v);
std::ostream & operator << (std::ostream & os, const ValueType t);
/**
* Initialise the Boehm GC, if applicable.
*/
void initGC();
struct RegexCache;
std::shared_ptr<RegexCache> makeRegexCache();
@ -167,13 +180,18 @@ struct DebugTrace {
class EvalState : public std::enable_shared_from_this<EvalState>
{
public:
const fetchers::Settings & fetchSettings;
const EvalSettings & settings;
SymbolTable symbols;
PosTable positions;
const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sRight, sWrong, sStructuredAttrs,
sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites,
sMaxSize, sMaxClosureSize,
sBuilder, sArgs,
sContentAddressed, sImpure,
sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations,
@ -276,6 +294,18 @@ public:
return std::shared_ptr<const StaticEnv>();;
}
/** Whether a debug repl can be started. If `false`, `runDebugRepl(error)` will return without starting a repl. */
bool canDebug();
/** Use front of `debugTraces`; see `runDebugRepl(error,env,expr)` */
void runDebugRepl(const Error * error);
/**
* Run a debug repl with the given error, environment and expression.
* @param error The error to debug, may be nullptr.
* @param env The environment to debug, matching the expression.
* @param expr The expression to debug, matching the environment.
*/
void runDebugRepl(const Error * error, const Env & env, const Expr & expr);
template<class T, typename... Args>
@ -294,15 +324,15 @@ private:
/* Cache for calls to addToStore(); maps source paths to the store
paths. */
std::map<SourcePath, StorePath> srcToStore;
Sync<std::unordered_map<SourcePath, StorePath>> srcToStore;
/**
* A cache from path names to parse trees.
*/
#if HAVE_BOEHMGC
typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
typedef std::unordered_map<SourcePath, Expr *, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
#else
typedef std::map<SourcePath, Expr *> FileParseCache;
typedef std::unordered_map<SourcePath, Expr *> FileParseCache;
#endif
FileParseCache fileParseCache;
@ -310,12 +340,18 @@ private:
* A cache from path names to values.
*/
#if HAVE_BOEHMGC
typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
typedef std::unordered_map<SourcePath, Value, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
#else
typedef std::map<SourcePath, Value> FileEvalCache;
typedef std::unordered_map<SourcePath, Value> FileEvalCache;
#endif
FileEvalCache fileEvalCache;
/**
* Associate source positions of certain AST nodes with their preceding doc comment, if they have one.
* Grouped by file.
*/
std::unordered_map<SourcePath, DocCommentMap> positionToDocComment;
LookupPath lookupPath;
std::map<std::string, std::optional<std::string>> lookupPathResolved;
@ -342,6 +378,8 @@ public:
EvalState(
const LookupPath & _lookupPath,
ref<Store> store,
const fetchers::Settings & fetchSettings,
const EvalSettings & settings,
std::shared_ptr<Store> buildStore = nullptr);
~EvalState();
@ -602,6 +640,12 @@ public:
const char * doc;
};
/**
* Retrieve the documentation for a value. This will evaluate the value if
* it is a thunk, and it will partially apply __functor if applicable.
*
* @param v The value to get the documentation for.
*/
std::optional<Doc> getDoc(Value & v);
private:
@ -626,12 +670,26 @@ private:
public:
/**
* Check that the call depth is within limits, and increment it, until the returned object is destroyed.
*/
inline CallDepth addCallDepth(const PosIdx pos);
/**
* Do a deep equality test between two values. That is, list
* elements and attributes are compared recursively.
*/
bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
/**
* Like `eqValues`, but throws an `AssertionError` if not equal.
*
* WARNING:
* Callers should call `eqValues` first and report if `assertEqValues` behaves
* incorrectly. (e.g. if it doesn't throw if eqValues returns false or vice versa)
*/
void assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
bool isFunctor(Value & fun);
// FIXME: use std::span
@ -756,6 +814,8 @@ public:
std::string_view pathArg,
PosIdx pos);
DocComment getDocCommentForPos(PosIdx pos);
private:
/**
@ -838,8 +898,10 @@ std::string showType(const Value & v);
/**
* If `path` refers to a directory, then append "/default.nix".
*
* @param addDefaultNix Whether to append "/default.nix" after resolving symlinks.
*/
SourcePath resolveExprPath(SourcePath path);
SourcePath resolveExprPath(SourcePath path, bool addDefaultNix = true);
/**
* Whether a URI is allowed, assuming restrictEval is enabled

81
src/libexpr/flake/config.cc
View file

@ -1,81 +0,0 @@
#include "users.hh"
#include "globals.hh"
#include "fetch-settings.hh"
#include "flake.hh"
#include <nlohmann/json.hpp>
namespace nix::flake {
// setting name -> setting value -> allow or ignore.
typedef std::map<std::string, std::map<std::string, bool>> TrustedList;
Path trustedListPath()
{
return getDataDir() + "/nix/trusted-settings.json";
}
static TrustedList readTrustedList()
{
auto path = trustedListPath();
if (!pathExists(path)) return {};
auto json = nlohmann::json::parse(readFile(path));
return json;
}
static void writeTrustedList(const TrustedList & trustedList)
{
auto path = trustedListPath();
createDirs(dirOf(path));
writeFile(path, nlohmann::json(trustedList).dump());
}
void ConfigFile::apply()
{
std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"};
for (auto & [name, value] : settings) {
auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name;
// FIXME: Move into libutil/config.cc.
std::string valueS;
if (auto* s = std::get_if<std::string>(&value))
valueS = *s;
else if (auto* n = std::get_if<int64_t>(&value))
valueS = fmt("%d", *n);
else if (auto* b = std::get_if<Explicit<bool>>(&value))
valueS = b->t ? "true" : "false";
else if (auto ss = std::get_if<std::vector<std::string>>(&value))
valueS = concatStringsSep(" ", *ss); // FIXME: evil
else
assert(false);
if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) {
bool trusted = false;
auto trustedList = readTrustedList();
auto tlname = get(trustedList, name);
if (auto saved = tlname ? get(*tlname, valueS) : nullptr) {
trusted = *saved;
printInfo("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name, valueS);
} else {
// FIXME: filter ANSI escapes, newlines, \r, etc.
if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') {
trusted = true;
}
if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?", trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') {
trustedList[name][valueS] = trusted;
writeTrustedList(trustedList);
}
}
if (!trusted) {
warn("ignoring untrusted flake configuration setting '%s'.\nPass '%s' to trust it", name, "--accept-flake-config");
continue;
}
}
globalConfig.set(name, valueS);
}
}
}

1044
src/libexpr/flake/flake.cc
View file

File diff suppressed because it is too large Load diff

216
src/libexpr/flake/flake.hh
View file

@ -1,216 +0,0 @@
#pragma once
///@file
#include "types.hh"
#include "flakeref.hh"
#include "lockfile.hh"
#include "value.hh"
namespace nix {
class EvalState;
namespace flake {
struct FlakeInput;
typedef std::map<FlakeId, FlakeInput> FlakeInputs;
/**
* FlakeInput is the 'Flake'-level parsed form of the "input" entries
* in the flake file.
*
* A FlakeInput is normally constructed by the 'parseFlakeInput'
* function which parses the input specification in the '.flake' file
* to create a 'FlakeRef' (a fetcher, the fetcher-specific
* representation of the input specification, and possibly the fetched
* local store path result) and then creating this FlakeInput to hold
* that FlakeRef, along with anything that might override that
* FlakeRef (like command-line overrides or "follows" specifications).
*
* A FlakeInput is also sometimes constructed directly from a FlakeRef
* instead of starting at the flake-file input specification
* (e.g. overrides, follows, and implicit inputs).
*
* A FlakeInput will usually have one of either "ref" or "follows"
* set. If not otherwise specified, a "ref" will be generated to a
* 'type="indirect"' flake, which is treated as simply the name of a
* flake to be resolved in the registry.
*/
struct FlakeInput
{
std::optional<FlakeRef> ref;
/**
* true = process flake to get outputs
*
* false = (fetched) static source path
*/
bool isFlake = true;
std::optional<InputPath> follows;
FlakeInputs overrides;
};
struct ConfigFile
{
using ConfigValue = std::variant<std::string, int64_t, Explicit<bool>, std::vector<std::string>>;
std::map<std::string, ConfigValue> settings;
void apply();
};
/**
* The contents of a flake.nix file.
*/
struct Flake
{
/**
* The original flake specification (by the user)
*/
FlakeRef originalRef;
/**
* registry references and caching resolved to the specific underlying flake
*/
FlakeRef resolvedRef;
/**
* the specific local store result of invoking the fetcher
*/
FlakeRef lockedRef;
/**
* The path of `flake.nix`.
*/
SourcePath path;
/**
* pretend that 'lockedRef' is dirty
*/
bool forceDirty = false;
std::optional<std::string> description;
FlakeInputs inputs;
/**
* 'nixConfig' attribute
*/
ConfigFile config;
~Flake();
SourcePath lockFilePath()
{
return path.parent() / "flake.lock";
}
};
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
/**
* Fingerprint of a locked flake; used as a cache key.
*/
typedef Hash Fingerprint;
struct LockedFlake
{
Flake flake;
LockFile lockFile;
/**
* Source tree accessors for nodes that have been fetched in
* lockFlake(); in particular, the root node and the overriden
* inputs.
*/
std::map<ref<Node>, SourcePath> nodePaths;
std::optional<Fingerprint> getFingerprint(ref<Store> store) const;
};
struct LockFlags
{
/**
* Whether to ignore the existing lock file, creating a new one
* from scratch.
*/
bool recreateLockFile = false;
/**
* Whether to update the lock file at all. If set to false, if any
* change to the lock file is needed (e.g. when an input has been
* added to flake.nix), you get a fatal error.
*/
bool updateLockFile = true;
/**
* Whether to write the lock file to disk. If set to true, if the
* any changes to the lock file are needed and the flake is not
* writable (i.e. is not a local Git working tree or similar), you
* get a fatal error. If set to false, Nix will use the modified
* lock file in memory only, without writing it to disk.
*/
bool writeLockFile = true;
/**
* Whether to use the registries to lookup indirect flake
* references like 'nixpkgs'.
*/
std::optional<bool> useRegistries = std::nullopt;
/**
* Whether to apply flake's nixConfig attribute to the configuration
*/
bool applyNixConfig = false;
/**
* Whether unlocked flake references (i.e. those without a Git
* revision or similar) without a corresponding lock are
* allowed. Unlocked flake references with a lock are always
* allowed.
*/
bool allowUnlocked = true;
/**
* Whether to commit changes to flake.lock.
*/
bool commitLockFile = false;
/**
* The path to a lock file to read instead of the `flake.lock` file in the top-level flake
*/
std::optional<SourcePath> referenceLockFilePath;
/**
* The path to a lock file to write to instead of the `flake.lock` file in the top-level flake
*/
std::optional<Path> outputLockFilePath;
/**
* Flake inputs to be overridden.
*/
std::map<InputPath, FlakeRef> inputOverrides;
/**
* Flake inputs to be updated. This means that any existing lock
* for those inputs will be ignored.
*/
std::set<InputPath> inputUpdates;
};
LockedFlake lockFlake(
EvalState & state,
const FlakeRef & flakeRef,
const LockFlags & lockFlags);
void callFlake(
EvalState & state,
const LockedFlake & lockedFlake,
Value & v);
}
void emitTreeAttrs(
EvalState & state,
const StorePath & storePath,
const fetchers::Input & input,
Value & v,
bool emptyRevFallback = false,
bool forceDirty = false);
}

312
src/libexpr/flake/flakeref.cc
View file

@ -1,312 +0,0 @@
#include "flakeref.hh"
#include "store-api.hh"
#include "url.hh"
#include "url-parts.hh"
#include "fetchers.hh"
#include "registry.hh"
namespace nix {
#if 0
// 'dir' path elements cannot start with a '.'. We also reject
// potentially dangerous characters like ';'.
const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
#endif
std::string FlakeRef::to_string() const
{
std::map<std::string, std::string> extraQuery;
if (subdir != "")
extraQuery.insert_or_assign("dir", subdir);
return input.toURLString(extraQuery);
}
fetchers::Attrs FlakeRef::toAttrs() const
{
auto attrs = input.toAttrs();
if (subdir != "")
attrs.emplace("dir", subdir);
return attrs;
}
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
{
str << flakeRef.to_string();
return str;
}
bool FlakeRef::operator ==(const FlakeRef & other) const
{
return input == other.input && subdir == other.subdir;
}
FlakeRef FlakeRef::resolve(ref<Store> store) const
{
auto [input2, extraAttrs] = lookupInRegistries(store, input);
return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
}
FlakeRef parseFlakeRef(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake,
bool allowRelative)
{
auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative);
if (fragment != "")
throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
return flakeRef;
}
std::optional<FlakeRef> maybeParseFlakeRef(
const std::string & url, const std::optional<Path> & baseDir)
{
try {
return parseFlakeRef(url, baseDir);
} catch (Error &) {
return {};
}
}
static std::pair<FlakeRef, std::string> fromParsedURL(
ParsedURL && parsedURL,
bool isFlake)
{
auto dir = getOr(parsedURL.query, "dir", "");
parsedURL.query.erase("dir");
std::string fragment;
std::swap(fragment, parsedURL.fragment);
return std::make_pair(FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), dir), fragment);
}
std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake,
bool allowRelative)
{
std::string path = url;
std::string fragment = "";
std::map<std::string, std::string> query;
auto pathEnd = url.find_first_of("#?");
auto fragmentStart = pathEnd;
if (pathEnd != std::string::npos && url[pathEnd] == '?') {
fragmentStart = url.find("#");
}
if (pathEnd != std::string::npos) {
path = url.substr(0, pathEnd);
}
if (fragmentStart != std::string::npos) {
fragment = percentDecode(url.substr(fragmentStart+1));
}
if (pathEnd != std::string::npos && fragmentStart != std::string::npos) {
query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1));
}
if (baseDir) {
/* Check if 'url' is a path (either absolute or relative
to 'baseDir'). If so, search upward to the root of the
repo (i.e. the directory containing .git). */
path = absPath(path, baseDir);
if (isFlake) {
if (!S_ISDIR(lstat(path).st_mode)) {
if (baseNameOf(path) == "flake.nix") {
// Be gentle with people who accidentally write `/foo/bar/flake.nix` instead of `/foo/bar`
warn(
"Path '%s' should point at the directory containing the 'flake.nix' file, not the file itself. "
"Pretending that you meant '%s'"
, path, dirOf(path));
path = dirOf(path);
} else {
throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
}
}
if (!allowMissing && !pathExists(path + "/flake.nix")){
notice("path '%s' does not contain a 'flake.nix', searching up",path);
// Save device to detect filesystem boundary
dev_t device = lstat(path).st_dev;
bool found = false;
while (path != "/") {
if (pathExists(path + "/flake.nix")) {
found = true;
break;
} else if (pathExists(path + "/.git"))
throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path);
else {
if (lstat(path).st_dev != device)
throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
}
path = dirOf(path);
}
if (!found)
throw BadURL("could not find a flake.nix file");
}
if (!allowMissing && !pathExists(path + "/flake.nix"))
throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
auto flakeRoot = path;
std::string subdir;
while (flakeRoot != "/") {
if (pathExists(flakeRoot + "/.git")) {
auto base = std::string("git+file://") + flakeRoot;
auto parsedURL = ParsedURL{
.url = base, // FIXME
.base = base,
.scheme = "git+file",
.authority = "",
.path = flakeRoot,
.query = query,
.fragment = fragment,
};
if (subdir != "") {
if (parsedURL.query.count("dir"))
throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
parsedURL.query.insert_or_assign("dir", subdir);
}
if (pathExists(flakeRoot + "/.git/shallow"))
parsedURL.query.insert_or_assign("shallow", "1");
return fromParsedURL(std::move(parsedURL), isFlake);
}
subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
flakeRoot = dirOf(flakeRoot);
}
}
} else {
if (!allowRelative && !hasPrefix(path, "/"))
throw BadURL("flake reference '%s' is not an absolute path", url);
}
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "path");
attrs.insert_or_assign("path", path);
return fromParsedURL({
.url = path,
.base = path,
.scheme = "path",
.authority = "",
.path = path,
.query = query,
.fragment = fragment
}, isFlake);
}
/* Check if 'url' is a flake ID. This is an abbreviated syntax for
'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
const std::string & url,
bool isFlake)
{
std::smatch match;
static std::regex flakeRegex(
"((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
+ "(?:#(" + fragmentRegex + "))?",
std::regex::ECMAScript);
if (std::regex_match(url, match, flakeRegex)) {
auto parsedURL = ParsedURL{
.url = url,
.base = "flake:" + match.str(1),
.scheme = "flake",
.authority = "",
.path = match[1],
};
return std::make_pair(
FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""),
percentDecode(match.str(6)));
}
return {};
}
std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
const std::string & url,
const std::optional<Path> & baseDir,
bool isFlake)
{
try {
return fromParsedURL(parseURL(url), isFlake);
} catch (BadURL &) {
return std::nullopt;
}
}
std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake,
bool allowRelative)
{
using namespace fetchers;
std::smatch match;
if (auto res = parseFlakeIdRef(url, isFlake)) {
return *res;
} else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) {
return *res;
} else {
return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative);
}
}
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir)
{
try {
return parseFlakeRefWithFragment(url, baseDir);
} catch (Error & e) {
return {};
}
}
FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
{
auto attrs2(attrs);
attrs2.erase("dir");
return FlakeRef(
fetchers::Input::fromAttrs(std::move(attrs2)),
fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
}
std::pair<StorePath, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
{
auto [storePath, lockedInput] = input.fetchToStore(store);
return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)};
}
std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake)
{
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url);
auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake);
return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)};
}
std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript);
}

115
src/libexpr/flake/flakeref.hh
View file

@ -1,115 +0,0 @@
#pragma once
///@file
#include "types.hh"
#include "hash.hh"
#include "fetchers.hh"
#include "outputs-spec.hh"
#include <regex>
#include <variant>
namespace nix {
class Store;
typedef std::string FlakeId;
/**
* A flake reference specifies how to fetch a flake or raw source
* (e.g. from a Git repository). It is created from a URL-like syntax
* (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
* type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
* path.
*
* Each flake will have a number of FlakeRef objects: one for each
* input to the flake.
*
* The normal method of constructing a FlakeRef is by starting with an
* input description (usually the attrs or a url from the flake file),
* locating a fetcher for that input, and then capturing the Input
* object that fetcher generates (usually via
* FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls).
*
* The actual fetch may not have been performed yet (i.e. a FlakeRef may
* be lazy), but the fetcher can be invoked at any time via the
* FlakeRef to ensure the store is populated with this input.
*/
struct FlakeRef
{
/**
* Fetcher-specific representation of the input, sufficient to
* perform the fetch operation.
*/
fetchers::Input input;
/**
* sub-path within the fetched input that represents this input
*/
Path subdir;
bool operator==(const FlakeRef & other) const;
FlakeRef(fetchers::Input && input, const Path & subdir)
: input(std::move(input)), subdir(subdir)
{ }
// FIXME: change to operator <<.
std::string to_string() const;
fetchers::Attrs toAttrs() const;
FlakeRef resolve(ref<Store> store) const;
static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
std::pair<StorePath, FlakeRef> fetchTree(ref<Store> store) const;
};
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
/**
* @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
*/
FlakeRef parseFlakeRef(
const std::string & url,
const std::optional<Path> & baseDir = {},
bool allowMissing = false,
bool isFlake = true,
bool allowRelative = false);
/**
* @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
*/
std::optional<FlakeRef> maybeParseFlake(
const std::string & url, const std::optional<Path> & baseDir = {});
/**
* @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
*/
std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
const std::string & url,
const std::optional<Path> & baseDir = {},
bool allowMissing = false,
bool isFlake = true,
bool allowRelative = false);
/**
* @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
*/
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir = {});
/**
* @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory)
*/
std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir = {},
bool allowMissing = false,
bool isFlake = true);
const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
extern std::regex flakeIdRegex;
}

377
src/libexpr/flake/lockfile.cc
View file

@ -1,377 +0,0 @@
#include "lockfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include <algorithm>
#include <iomanip>
#include <iterator>
#include <nlohmann/json.hpp>
namespace nix::flake {
FlakeRef getFlakeRef(
const nlohmann::json & json,
const char * attr,
const char * info)
{
auto i = json.find(attr);
if (i != json.end()) {
auto attrs = fetchers::jsonToAttrs(*i);
// FIXME: remove when we drop support for version 5.
if (info) {
auto j = json.find(info);
if (j != json.end()) {
for (auto k : fetchers::jsonToAttrs(*j))
attrs.insert_or_assign(k.first, k.second);
}
}
return FlakeRef::fromAttrs(attrs);
}
throw Error("attribute '%s' missing in lock file", attr);
}
LockedNode::LockedNode(const nlohmann::json & json)
: lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info"
, originalRef(getFlakeRef(json, "original", nullptr))
, isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
, parentPath(json.find("parent") != json.end() ? (std::optional<InputPath>) json["parent"] : std::nullopt)
{
if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative())
throw Error("lock file contains unlocked input '%s'",
fetchers::attrsToJSON(lockedRef.input.toAttrs()));
}
StorePath LockedNode::computeStorePath(Store & store) const
{
return lockedRef.input.computeStorePath(store);
}
static std::shared_ptr<Node> doFind(const ref<Node>& root, const InputPath & path, std::vector<InputPath>& visited) {
auto pos = root;
auto found = std::find(visited.cbegin(), visited.cend(), path);
if(found != visited.end()) {
std::vector<std::string> cycle;
std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath);
cycle.push_back(printInputPath(path));
throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle));
}
visited.push_back(path);
for (auto & elem : path) {
if (auto i = get(pos->inputs, elem)) {
if (auto node = std::get_if<0>(&*i))
pos = *node;
else if (auto follows = std::get_if<1>(&*i)) {
if (auto p = doFind(root, *follows, visited))
pos = ref(p);
else
return {};
}
} else
return {};
}
return pos;
}
std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
{
std::vector<InputPath> visited;
return doFind(root, path, visited);
}
LockFile::LockFile(std::string_view contents, std::string_view path)
{
auto json = nlohmann::json::parse(contents);
auto version = json.value("version", 0);
if (version < 5 || version > 7)
throw Error("lock file '%s' has unsupported version %d", path, version);
std::map<std::string, ref<Node>> nodeMap;
std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs;
getInputs = [&](Node & node, const nlohmann::json & jsonNode)
{
if (jsonNode.find("inputs") == jsonNode.end()) return;
for (auto & i : jsonNode["inputs"].items()) {
if (i.value().is_array()) { // FIXME: remove, obsolete
InputPath path;
for (auto & j : i.value())
path.push_back(j);
node.inputs.insert_or_assign(i.key(), path);
} else {
std::string inputKey = i.value();
auto k = nodeMap.find(inputKey);
if (k == nodeMap.end()) {
auto & nodes = json["nodes"];
auto jsonNode2 = nodes.find(inputKey);
if (jsonNode2 == nodes.end())
throw Error("lock file references missing node '%s'", inputKey);
auto input = make_ref<LockedNode>(*jsonNode2);
k = nodeMap.insert_or_assign(inputKey, input).first;
getInputs(*input, *jsonNode2);
}
if (auto child = k->second.dynamic_pointer_cast<LockedNode>())
node.inputs.insert_or_assign(i.key(), ref(child));
else
// FIXME: replace by follows node
throw Error("lock file contains cycle to root node");
}
}
};
std::string rootKey = json["root"];
nodeMap.insert_or_assign(rootKey, root);
getInputs(*root, json["nodes"][rootKey]);
// FIXME: check that there are no cycles in version >= 7. Cycles
// between inputs are only possible using 'follows' indirections.
// Once we drop support for version <= 6, we can simplify the code
// a bit since we don't need to worry about cycles.
}
std::pair<nlohmann::json, LockFile::KeyMap> LockFile::toJSON() const
{
nlohmann::json nodes;
KeyMap nodeKeys;
std::unordered_set<std::string> keys;
std::function<std::string(const std::string & key, ref<const Node> node)> dumpNode;
dumpNode = [&](std::string key, ref<const Node> node) -> std::string
{
auto k = nodeKeys.find(node);
if (k != nodeKeys.end())
return k->second;
if (!keys.insert(key).second) {
for (int n = 2; ; ++n) {
auto k = fmt("%s_%d", key, n);
if (keys.insert(k).second) {
key = k;
break;
}
}
}
nodeKeys.insert_or_assign(node, key);
auto n = nlohmann::json::object();
if (!node->inputs.empty()) {
auto inputs = nlohmann::json::object();
for (auto & i : node->inputs) {
if (auto child = std::get_if<0>(&i.second)) {
inputs[i.first] = dumpNode(i.first, *child);
} else if (auto follows = std::get_if<1>(&i.second)) {
auto arr = nlohmann::json::array();
for (auto & x : *follows)
arr.push_back(x);
inputs[i.first] = std::move(arr);
}
}
n["inputs"] = std::move(inputs);
}
if (auto lockedNode = node.dynamic_pointer_cast<const LockedNode>()) {
n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs());
n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs());
if (!lockedNode->isFlake)
n["flake"] = false;
if (lockedNode->parentPath)
n["parent"] = *lockedNode->parentPath;
}
nodes[key] = std::move(n);
return key;
};
nlohmann::json json;
json["version"] = 7;
json["root"] = dumpNode("root", root);
json["nodes"] = std::move(nodes);
return {json, std::move(nodeKeys)};
}
std::pair<std::string, LockFile::KeyMap> LockFile::to_string() const
{
auto [json, nodeKeys] = toJSON();
return {json.dump(2), std::move(nodeKeys)};
}
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
{
stream << lockFile.toJSON().first.dump(2);
return stream;
}
std::optional<FlakeRef> LockFile::isUnlocked() const
{
std::set<ref<const Node>> nodes;
std::function<void(ref<const Node> node)> visit;
visit = [&](ref<const Node> node)
{
if (!nodes.insert(node).second) return;
for (auto & i : node->inputs)
if (auto child = std::get_if<0>(&i.second))
visit(*child);
};
visit(root);
for (auto & i : nodes) {
if (i == ref<const Node>(root)) continue;
auto node = i.dynamic_pointer_cast<const LockedNode>();
if (node
&& !node->lockedRef.input.isLocked()
&& !node->lockedRef.input.isRelative())
return node->lockedRef;
}
return {};
}
bool LockFile::operator ==(const LockFile & other) const
{
// FIXME: slow
return toJSON().first == other.toJSON().first;
}
bool LockFile::operator !=(const LockFile & other) const
{
return !(*this == other);
}
InputPath parseInputPath(std::string_view s)
{
InputPath path;
for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
if (!std::regex_match(elem, flakeIdRegex))
throw UsageError("invalid flake input path element '%s'", elem);
path.push_back(elem);
}
return path;
}
std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
{
std::set<ref<Node>> done;
std::map<InputPath, Node::Edge> res;
std::function<void(const InputPath & prefix, ref<Node> node)> recurse;
recurse = [&](const InputPath & prefix, ref<Node> node)
{
if (!done.insert(node).second) return;
for (auto &[id, input] : node->inputs) {
auto inputPath(prefix);
inputPath.push_back(id);
res.emplace(inputPath, input);
if (auto child = std::get_if<0>(&input))
recurse(inputPath, *child);
}
};
recurse({}, root);
return res;
}
static std::string describe(const FlakeRef & flakeRef)
{
auto s = fmt("'%s'", flakeRef.to_string());
if (auto lastModified = flakeRef.input.getLastModified())
s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d"));
return s;
}
std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
{
if (auto node = std::get_if<0>(&edge))
stream << describe((*node)->lockedRef);
else if (auto follows = std::get_if<1>(&edge))
stream << fmt("follows '%s'", printInputPath(*follows));
return stream;
}
static bool equals(const Node::Edge & e1, const Node::Edge & e2)
{
if (auto n1 = std::get_if<0>(&e1))
if (auto n2 = std::get_if<0>(&e2))
return (*n1)->lockedRef == (*n2)->lockedRef;
if (auto f1 = std::get_if<1>(&e1))
if (auto f2 = std::get_if<1>(&e2))
return *f1 == *f2;
return false;
}
std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
{
auto oldFlat = oldLocks.getAllInputs();
auto newFlat = newLocks.getAllInputs();
auto i = oldFlat.begin();
auto j = newFlat.begin();
std::string res;
while (i != oldFlat.end() || j != newFlat.end()) {
if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
res += fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n",
printInputPath(j->first), j->second);
++j;
} else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputPath(i->first));
++i;
} else {
if (!equals(i->second, j->second)) {
res += fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n",
printInputPath(i->first),
i->second,
j->second);
}
++i;
++j;
}
}
return res;
}
void LockFile::check()
{
auto inputs = getAllInputs();
for (auto & [inputPath, input] : inputs) {
if (auto follows = std::get_if<1>(&input)) {
if (!follows->empty() && !findInput(*follows))
throw Error("input '%s' follows a non-existent input '%s'",
printInputPath(inputPath),
printInputPath(*follows));
}
}
}
void check();
std::string printInputPath(const InputPath & path)
{
return concatStringsSep("/", path);
}
}

103
src/libexpr/flake/lockfile.hh
View file

@ -1,103 +0,0 @@
#pragma once
///@file
#include "flakeref.hh"
#include <nlohmann/json_fwd.hpp>
namespace nix {
class Store;
class StorePath;
}
namespace nix::flake {
typedef std::vector<FlakeId> InputPath;
struct LockedNode;
/**
* A node in the lock file. It has outgoing edges to other nodes (its
* inputs). Only the root node has this type; all other nodes have
* type LockedNode.
*/
struct Node : std::enable_shared_from_this<Node>
{
typedef std::variant<ref<LockedNode>, InputPath> Edge;
std::map<FlakeId, Edge> inputs;
virtual ~Node() { }
};
/**
* A non-root node in the lock file.
*/
struct LockedNode : Node
{
FlakeRef lockedRef, originalRef;
bool isFlake = true;
/* The node relative to which relative source paths
(e.g. 'path:../foo') are interpreted. */
std::optional<InputPath> parentPath;
LockedNode(
const FlakeRef & lockedRef,
const FlakeRef & originalRef,
bool isFlake = true,
std::optional<InputPath> parentPath = {})
: lockedRef(lockedRef)
, originalRef(originalRef)
, isFlake(isFlake)
, parentPath(parentPath)
{ }
LockedNode(const nlohmann::json & json);
StorePath computeStorePath(Store & store) const;
};
struct LockFile
{
ref<Node> root = make_ref<Node>();
LockFile() {};
LockFile(std::string_view contents, std::string_view path);
typedef std::map<ref<const Node>, std::string> KeyMap;
std::pair<nlohmann::json, KeyMap> toJSON() const;
std::pair<std::string, KeyMap> to_string() const;
/**
* Check whether this lock file has any unlocked inputs. If so,
* return one.
*/
std::optional<FlakeRef> isUnlocked() const;
bool operator ==(const LockFile & other) const;
// Needed for old gcc versions that don't synthesize it (like gcc 8.2.2
// that is still the default on aarch64-linux)
bool operator !=(const LockFile & other) const;
std::shared_ptr<Node> findInput(const InputPath & path);
std::map<InputPath, Node::Edge> getAllInputs() const;
static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
/**
* Check that every 'follows' input target exists.
*/
void check();
};
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
InputPath parseInputPath(std::string_view s);
std::string printInputPath(const InputPath & path);
}

46
src/libexpr/flake/url-name.cc
View file

@ -1,46 +0,0 @@
#include "url-name.hh"
#include <regex>
#include <iostream>
namespace nix {
static const std::string attributeNamePattern("[a-zA-Z0-9_-]+");
static const std::regex lastAttributeRegex("^((?:" + attributeNamePattern + "\\.)*)(" + attributeNamePattern +")(\\^.*)?$");
static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?");
static const std::regex gitProviderRegex("github|gitlab|sourcehut");
static const std::regex gitSchemeRegex("git($|\\+.*)");
std::optional<std::string> getNameFromURL(const ParsedURL & url)
{
std::smatch match;
/* If there is a dir= argument, use its value */
if (url.query.count("dir") > 0)
return url.query.at("dir");
/* If the fragment isn't a "default" and contains two attribute elements, use the last one */
if (std::regex_match(url.fragment, match, lastAttributeRegex)
&& match.str(1) != "defaultPackage."
&& match.str(2) != "default") {
return match.str(2);
}
/* If this is a github/gitlab/sourcehut flake, use the repo name */
if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex))
return match.str(1);
/* If it is a regular git flake, use the directory name */
if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex))
return match.str(1);
/* If there is no fragment, take the last element of the path */
if (std::regex_match(url.path, match, lastPathSegmentRegex))
return match.str(1);
/* If even that didn't work, the URL does not contain enough info to determine a useful name */
return {};
}
}

20
src/libexpr/flake/url-name.hh
View file

@ -1,20 +0,0 @@
#include "url.hh"
#include "url-parts.hh"
#include "util.hh"
#include "split.hh"
namespace nix {
/**
* Try to extract a reasonably unique and meaningful, human-readable
* name of a flake output from a parsed URL.
* When nullopt is returned, the callsite should use information available
* to it outside of the URL to determine a useful name.
* This is a heuristic approach intended for user interfaces.
* @return nullopt if the extracted name is not useful to identify a
* flake output, for example because it is empty or "default".
* Otherwise returns the extracted name.
*/
std::optional<std::string> getNameFromURL(const ParsedURL & url);
}

56
src/libexpr/get-drvs.cc
View file

@ -69,10 +69,17 @@ std::string PackageInfo::querySystem() const
std::optional<StorePath> PackageInfo::queryDrvPath() const
{
if (!drvPath && attrs) {
NixStringContext context;
if (auto i = attrs->get(state->sDrvPath))
drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")};
else
if (auto i = attrs->get(state->sDrvPath)) {
NixStringContext context;
auto found = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
try {
found.requireDerivation();
} catch (Error & e) {
e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
throw;
}
drvPath = {std::move(found)};
} else
drvPath = {std::nullopt};
}
return drvPath.value_or(std::nullopt);
@ -239,8 +246,8 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def)
if (v->type() == nString) {
/* Backwards compatibility with before we had support for
integer meta fields. */
if (auto n = string2Int<NixInt>(v->c_str()))
return *n;
if (auto n = string2Int<NixInt::Inner>(v->c_str()))
return NixInt{*n};
}
return def;
}
@ -335,9 +342,9 @@ std::optional<PackageInfo> getDerivation(EvalState & state, Value & v,
}
static std::string addToPath(const std::string & s1, const std::string & s2)
static std::string addToPath(const std::string & s1, std::string_view s2)
{
return s1.empty() ? s2 : s1 + "." + s2;
return s1.empty() ? std::string(s2) : s1 + "." + s2;
}
@ -367,21 +374,26 @@ static void getDerivations(EvalState & state, Value & vIn,
bound to the attribute with the "lower" name should take
precedence). */
for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
debug("evaluating attribute '%1%'", state.symbols[i->name]);
if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
continue;
std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
if (combineChannels)
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
/* If the value of this attribute is itself a set,
should we recurse into it? => Only if it has a
`recurseForDerivations = true' attribute. */
if (i->value->type() == nAttrs) {
auto j = i->value->attrs()->get(state.sRecurseForDerivations);
if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
try {
debug("evaluating attribute '%1%'", state.symbols[i->name]);
if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
continue;
std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
if (combineChannels)
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
/* If the value of this attribute is itself a set,
should we recurse into it? => Only if it has a
`recurseForDerivations = true' attribute. */
if (i->value->type() == nAttrs) {
auto j = i->value->attrs()->get(state.sRecurseForDerivations);
if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
}
}
} catch (Error & e) {
e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", state.symbols[i->name]);
throw;
}
}
}

33
src/libexpr/json-to-value.cc
View file

@ -2,6 +2,7 @@
#include "value.hh"
#include "eval.hh"
#include <limits>
#include <variant>
#include <nlohmann/json.hpp>
@ -42,7 +43,7 @@ class JSONSax : nlohmann::json_sax<json> {
auto attrs2 = state.buildBindings(attrs.size());
for (auto & i : attrs)
attrs2.insert(i.first, i.second);
parent->value(state).mkAttrs(attrs2.alreadySorted());
parent->value(state).mkAttrs(attrs2);
return std::move(parent);
}
void add() override { v = nullptr; }
@ -80,42 +81,46 @@ class JSONSax : nlohmann::json_sax<json> {
public:
JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
bool null()
bool null() override
{
rs->value(state).mkNull();
rs->add();
return true;
}
bool boolean(bool val)
bool boolean(bool val) override
{
rs->value(state).mkBool(val);
rs->add();
return true;
}
bool number_integer(number_integer_t val)
bool number_integer(number_integer_t val) override
{
rs->value(state).mkInt(val);
rs->add();
return true;
}
bool number_unsigned(number_unsigned_t val)
bool number_unsigned(number_unsigned_t val_) override
{
if (val_ > std::numeric_limits<NixInt::Inner>::max()) {
throw Error("unsigned json number %1% outside of Nix integer range", val_);
}
NixInt::Inner val = val_;
rs->value(state).mkInt(val);
rs->add();
return true;
}
bool number_float(number_float_t val, const string_t & s)
bool number_float(number_float_t val, const string_t & s) override
{
rs->value(state).mkFloat(val);
rs->add();
return true;
}
bool string(string_t & val)
bool string(string_t & val) override
{
rs->value(state).mkString(val);
rs->add();
@ -123,7 +128,7 @@ public:
}
#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
bool binary(binary_t&)
bool binary(binary_t&) override
{
// This function ought to be unreachable
assert(false);
@ -131,35 +136,35 @@ public:
}
#endif
bool start_object(std::size_t len)
bool start_object(std::size_t len) override
{
rs = std::make_unique<JSONObjectState>(std::move(rs));
return true;
}
bool key(string_t & name)
bool key(string_t & name) override
{
dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
return true;
}
bool end_object() {
bool end_object() override {
rs = rs->resolve(state);
rs->add();
return true;
}
bool end_array() {
bool end_array() override {
return end_object();
}
bool start_array(size_t len) {
bool start_array(size_t len) override {
rs = std::make_unique<JSONListState>(std::move(rs),
len != std::numeric_limits<size_t>::max() ? len : 128);
return true;
}
bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override {
throw JSONParseError("%s", ex.what());
}
};

30
src/libexpr/lexer-helpers.cc Normal file
View file

@ -0,0 +1,30 @@
#include "lexer-tab.hh"
#include "lexer-helpers.hh"
#include "parser-tab.hh"
void nix::lexer::internal::initLoc(YYLTYPE * loc)
{
loc->beginOffset = loc->endOffset = 0;
}
void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
{
loc->stash();
LexerState & lexerState = *yyget_extra(yyscanner);
if (lexerState.docCommentDistance == 1) {
// Preceding token was a doc comment.
ParserLocation doc;
doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset;
ParserLocation docEnd;
docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset;
DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)};
PosIdx locPos = lexerState.at(*loc);
lexerState.positionToDocComment.emplace(locPos, docComment);
}
lexerState.docCommentDistance++;
loc->beginOffset = loc->endOffset;
loc->endOffset += len;
}

9
src/libexpr/lexer-helpers.hh Normal file
View file

@ -0,0 +1,9 @@
#pragma once
namespace nix::lexer::internal {
void initLoc(YYLTYPE * loc);
void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
} // namespace nix::lexer

81
src/libexpr/lexer.l
View file

@ -5,7 +5,7 @@
%option stack
%option nodefault
%option nounput noyy_top_state
%option extra-type="::nix::LexerState *"
%s DEFAULT
%x STRING
@ -14,38 +14,31 @@
%x INPATH_SLASH
%x PATH_START
%top {
#include "parser-tab.hh" // YYSTYPE
#include "parser-state.hh"
}
%{
#ifdef __clang__
#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
#endif
#include <boost/lexical_cast.hpp>
#include "nixexpr.hh"
#include "parser-tab.hh"
#include "lexer-helpers.hh"
namespace nix {
struct LexerState;
}
using namespace nix;
using namespace nix::lexer::internal;
namespace nix {
#define CUR_POS state->at(*yylloc)
static void initLoc(YYLTYPE * loc)
{
loc->first_line = loc->last_line = 0;
loc->first_column = loc->last_column = 0;
}
static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
{
loc->stash();
loc->first_column = loc->last_column;
loc->last_column += len;
}
// we make use of the fact that the parser receives a private copy of the input
// string and can munge around in it.
static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
@ -74,6 +67,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
return {result, size_t(t - result)};
}
static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos)
{
if (!experimentalFeatureSettings.isEnabled(feature))
throw ParseError(ErrorInfo{
.msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)),
.pos = pos,
});
}
}
@ -81,7 +82,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
#define YY_USER_INIT initLoc(yylloc)
#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng);
#define PUSH_STATE(state) yy_push_state(state, yyscanner)
#define POP_STATE() yy_pop_state(yyscanner)
@ -126,12 +127,19 @@ or { return OR_KW; }
\-\> { return IMPL; }
\/\/ { return UPDATE; }
\+\+ { return CONCAT; }
\<\| { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]);
return PIPE_FROM;
}
\|\> { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]);
return PIPE_INTO;
}
{ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; }
{INT} { errno = 0;
try {
yylval->n = boost::lexical_cast<int64_t>(yytext);
} catch (const boost::bad_lexical_cast &) {
std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
if (numMay.has_value()) {
yylval->n = NixInt{*numMay};
} else {
throw ParseError(ErrorInfo{
.msg = HintFmt("invalid integer '%1%'", yytext),
.pos = state->positions[CUR_POS],
@ -280,9 +288,32 @@ or { return OR_KW; }
{SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
{URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
[ \t\r\n]+ /* eat up whitespace */
\#[^\r\n]* /* single-line comments */
\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */
%{
// Doc comment rule
//
// \/\*\* /**
// [^/*] reject /**/ (empty comment) and /***
// ([^*]|\*+[^*/])*\*+\/ same as the long comment rule
// ( )* zero or more non-ending sequences
// \* end(1)
// \/ end(2)
%}
\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/ /* doc comments */ {
LexerState & lexerState = *yyget_extra(yyscanner);
lexerState.docCommentDistance = 0;
lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset;
lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset;
}
%{
// The following rules have docCommentDistance--
// This compensates for the docCommentDistance++ which happens by default to
// make all the other rules invalidate the doc comment.
%}
[ \t\r\n]+ /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; }
\#[^\r\n]* /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
{ANY} {
/* Don't return a negative number, as this will cause

11
src/libexpr/local.mk
View file

@ -8,14 +8,15 @@ libexpr_SOURCES := \
$(wildcard $(d)/*.cc) \
$(wildcard $(d)/value/*.cc) \
$(wildcard $(d)/primops/*.cc) \
$(wildcard $(d)/flake/*.cc) \
$(d)/lexer-tab.cc \
$(d)/parser-tab.cc
# Not just for this library itself, but also for downstream libraries using this library
INCLUDE_libexpr := -I $(d)
libexpr_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libmain) $(INCLUDE_libexpr)
libexpr_CXXFLAGS += \
$(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) \
-DGC_THREADS
libexpr_LIBS = libutil libstore libfetchers
@ -43,11 +44,7 @@ $(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconf
$(foreach i, $(wildcard src/libexpr/value/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh
$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM =
$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/call-flake.nix.gen.hh

210
src/libexpr/meson.build Normal file
View file

@ -0,0 +1,210 @@
project('nix-expr', 'cpp',
version : files('.version'),
default_options : [
'cpp_std=c++2a',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'debug=true',
'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
license : 'LGPL-2.1-or-later',
)
cxx = meson.get_compiler('cpp')
subdir('build-utils-meson/deps-lists')
configdata = configuration_data()
deps_private_maybe_subproject = [
]
deps_public_maybe_subproject = [
dependency('nix-util'),
dependency('nix-store'),
dependency('nix-fetchers'),
]
subdir('build-utils-meson/subprojects')
subdir('build-utils-meson/threads')
boost = dependency(
'boost',
modules : ['container', 'context'],
include_type: 'system',
)
# boost is a public dependency, but not a pkg-config dependency unfortunately, so we
# put in `deps_other`.
deps_other += boost
nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
deps_public += nlohmann_json
bdw_gc = dependency('bdw-gc', required : get_option('gc'))
if bdw_gc.found()
deps_public += bdw_gc
foreach funcspec : [
'pthread_attr_get_np',
'pthread_getattr_np',
]
define_name = 'HAVE_' + funcspec.underscorify().to_upper()
define_value = cxx.has_function(funcspec).to_int()
configdata.set(define_name, define_value)
endforeach
configdata.set('GC_THREADS', 1)
endif
configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int())
toml11 = dependency(
'toml11',
version : '>=3.7.0',
method : 'cmake',
include_type: 'system',
)
deps_other += toml11
config_h = configure_file(
configuration : configdata,
output : 'config-expr.hh',
)
add_project_arguments(
# TODO(Qyriad): Yes this is how the autoconf+Make system did it.
# It would be nice for our headers to be idempotent instead.
'-include', 'config-util.hh',
'-include', 'config-store.hh',
# '-include', 'config-fetchers.h',
'-include', 'config-expr.hh',
language : 'cpp',
)
subdir('build-utils-meson/diagnostics')
parser_tab = custom_target(
input : 'parser.y',
output : [
'parser-tab.cc',
'parser-tab.hh',
],
command : [
'bison',
'-v',
'-o',
'@OUTPUT0@',
'@INPUT@',
'-d',
],
# NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
# an install script below which removes parser-tab.cc.
install : true,
install_dir : get_option('includedir') / 'nix',
)
lexer_tab = custom_target(
input : [
'lexer.l',
parser_tab,
],
output : [
'lexer-tab.cc',
'lexer-tab.hh',
],
command : [
'flex',
'--outfile',
'@OUTPUT0@',
'--header-file=' + '@OUTPUT1@',
'@INPUT0@',
],
# NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
# an install script below which removes lexer-tab.cc.
install : true,
install_dir : get_option('includedir') / 'nix',
)
subdir('build-utils-meson/generate-header')
generated_headers = []
foreach header : [
'imported-drv-to-derivation.nix',
'fetchurl.nix',
'call-flake.nix',
]
generated_headers += gen_header.process(header)
endforeach
sources = files(
'attr-path.cc',
'attr-set.cc',
'eval-cache.cc',
'eval-error.cc',
'eval-gc.cc',
'eval-settings.cc',
'eval.cc',
'function-trace.cc',
'get-drvs.cc',
'json-to-value.cc',
'lexer-helpers.cc',
'nixexpr.cc',
'paths.cc',
'primops.cc',
'print-ambiguous.cc',
'print.cc',
'search-path.cc',
'value-to-json.cc',
'value-to-xml.cc',
'value/context.cc',
)
include_dirs = [include_directories('.')]
headers = [config_h] + files(
'attr-path.hh',
'attr-set.hh',
'eval-cache.hh',
'eval-error.hh',
'eval-gc.hh',
'eval-inline.hh',
'eval-settings.hh',
'eval.hh',
'function-trace.hh',
'gc-small-vector.hh',
'get-drvs.hh',
'json-to-value.hh',
# internal: 'lexer-helpers.hh',
'nixexpr.hh',
'parser-state.hh',
'pos-idx.hh',
'pos-table.hh',
'primops.hh',
'print-ambiguous.hh',
'print-options.hh',
'print.hh',
'repl-exit-status.hh',
'search-path.hh',
'symbol-table.hh',
'value-to-json.hh',
'value-to-xml.hh',
'value.hh',
'value/context.hh',
)
subdir('primops')
this_library = library(
'nixexpr',
sources,
parser_tab,
lexer_tab,
generated_headers,
dependencies : deps_public + deps_private + deps_other,
prelink : true, # For C++ static initializers
install : true,
)
install_headers(headers, subdir : 'nix', preserve_path : true)
libraries_private = []
subdir('build-utils-meson/export')

3
src/libexpr/meson.options Normal file
View file

@ -0,0 +1,3 @@
option('gc', type : 'feature',
description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)',
)

50
src/libexpr/nixexpr.cc
View file

@ -1,11 +1,13 @@
#include "nixexpr.hh"
#include "derivations.hh"
#include "eval.hh"
#include "symbol-table.hh"
#include "util.hh"
#include "print.hh"
#include <cstdlib>
#include <sstream>
#include "strings-inline.hh"
namespace nix {
@ -23,7 +25,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
void Expr::show(const SymbolTable & symbols, std::ostream & str) const
{
abort();
unreachable();
}
void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const
@ -80,7 +82,9 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
return sa < sb;
});
std::vector<Symbol> inherits;
std::map<ExprInheritFrom *, std::vector<Symbol>> inheritsFrom;
// We can use the displacement as a proxy for the order in which the symbols were parsed.
// The assignment of displacements should be deterministic, so that showBindings is deterministic.
std::map<Displacement, std::vector<Symbol>> inheritsFrom;
for (auto & i : sorted) {
switch (i->second.kind) {
case AttrDef::Kind::Plain:
@ -91,7 +95,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
case AttrDef::Kind::InheritedFrom: {
auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
inheritsFrom[&from].push_back(i->first);
inheritsFrom[from.displ].push_back(i->first);
break;
}
}
@ -103,7 +107,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
}
for (const auto & [from, syms] : inheritsFrom) {
str << "inherit (";
(*inheritFromExprs)[from->displ]->show(symbols, str);
(*inheritFromExprs)[from]->show(symbols, str);
str << ")";
for (auto sym : syms) str << " " << symbols[sym];
str << "; ";
@ -267,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
void Expr::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{
abort();
unreachable();
}
void ExprInt::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
@ -581,6 +585,22 @@ std::string ExprLambda::showNamePos(const EvalState & state) const
return fmt("%1% at %2%", id, state.positions[pos]);
}
void ExprLambda::setDocComment(DocComment docComment) {
// RFC 145 specifies that the innermost doc comment wins.
// See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement
if (!this->docComment) {
this->docComment = docComment;
// Curried functions are defined by putting a function directly
// in the body of another function. To render docs for those, we
// need to propagate the doc comment to the innermost function.
//
// If we have our own comment, we've already propagated it, so this
// belongs in the same conditional.
body->setDocComment(docComment);
}
};
/* Position table. */
@ -625,4 +645,22 @@ size_t SymbolTable::totalSize() const
return n;
}
std::string DocComment::getInnerText(const PosTable & positions) const {
auto beginPos = positions[begin];
auto endPos = positions[end];
auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or("");
// Strip "/**" and "*/"
constexpr size_t prefixLen = 3;
constexpr size_t suffixLen = 2;
std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen);
if (docStr.empty())
return {};
// Turn the now missing "/**" into indentation
docStr = " " + docStr;
// Strip indentation (for the whole, potentially multi-line string)
docStr = stripIndentation(docStr);
return docStr;
}
}

67
src/libexpr/nixexpr.hh
View file

@ -6,21 +6,58 @@
#include "value.hh"
#include "symbol-table.hh"
#include "error.hh"
#include "position.hh"
#include "eval-error.hh"
#include "pos-idx.hh"
#include "pos-table.hh"
namespace nix {
struct Env;
struct Value;
class EvalState;
class PosTable;
struct Env;
struct ExprWith;
struct StaticEnv;
struct Value;
/**
* A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md)
*
* Note that this does not implement the following:
* - argument attribute names ("formals"): TBD
* - argument names: these are internal to the function and their names may not be optimal for documentation
* - function arity (degree of currying or number of ':'s):
* - Functions returning partially applied functions have a higher arity
* than can be determined locally and without evaluation.
* We do not want to present false data.
* - Some functions should be thought of as transformations of other
* functions. For instance `overlay -> overlay -> overlay` is the simplest
* way to understand `composeExtensions`, but its implementation looks like
* `f: g: final: prev: <...>`. The parameters `final` and `prev` are part
* of the overlay concept, while distracting from the function's purpose.
*/
struct DocComment {
/**
* Start of the comment, including the opening, ie `/` and `**`.
*/
PosIdx begin;
/**
* Position right after the final asterisk and `/` that terminate the comment.
*/
PosIdx end;
/**
* Whether the comment is set.
*
* A `DocComment` is small enough that it makes sense to pass by value, and
* therefore baking optionality into it is also useful, to avoiding the memory
* overhead of `std::optional`.
*/
operator bool() const { return static_cast<bool>(begin); }
std::string getInnerText(const PosTable & positions) const;
};
/**
* An attribute path is a sequence of attribute names.
@ -57,6 +94,7 @@ struct Expr
virtual void eval(EvalState & state, Env & env, Value & v);
virtual Value * maybeThunk(EvalState & state, Env & env);
virtual void setName(Symbol name);
virtual void setDocComment(DocComment docComment) { };
virtual PosIdx getPos() const { return noPos; }
};
@ -69,6 +107,7 @@ struct ExprInt : Expr
{
Value v;
ExprInt(NixInt n) { v.mkInt(n); };
ExprInt(NixInt::Inner n) { v.mkInt(n); };
Value * maybeThunk(EvalState & state, Env & env) override;
COMMON_METHODS
};
@ -148,7 +187,7 @@ struct ExprInheritFrom : ExprVar
this->fromWith = nullptr;
}
void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override;
};
struct ExprSelect : Expr
@ -159,6 +198,17 @@ struct ExprSelect : Expr
ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
PosIdx getPos() const override { return pos; }
/**
* Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl.
*
* @param[out] v The attribute set that should contain the last attribute name (if it exists).
* @return The last attribute name in `attrPath`
*
* @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist.
*/
Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs);
COMMON_METHODS
};
@ -281,6 +331,8 @@ struct ExprLambda : Expr
Symbol arg;
Formals * formals;
Expr * body;
DocComment docComment;
ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
: pos(pos), arg(arg), formals(formals), body(body)
{
@ -293,6 +345,7 @@ struct ExprLambda : Expr
std::string showNamePos(const EvalState & state) const;
inline bool hasFormals() const { return formals != nullptr; }
PosIdx getPos() const override { return pos; }
virtual void setDocComment(DocComment docComment) override;
COMMON_METHODS
};

113
src/libexpr/package.nix Normal file
View file

@ -0,0 +1,113 @@
{ lib
, stdenv
, mkMesonDerivation
, releaseTools
, meson
, ninja
, pkg-config
, bison
, flex
, cmake # for resolving toml11 dep
, nix-util
, nix-store
, nix-fetchers
, boost
, boehmgc
, nlohmann_json
, toml11
# Configuration Options
, version
# Whether to use garbage collection for the Nix language evaluator.
#
# If it is disabled, we just leak memory, but this is not as bad as it
# sounds so long as evaluation just takes places within short-lived
# processes. (When the process exits, the memory is reclaimed; it is
# only leaked *within* the process.)
#
# Temporarily disabled on Windows because the `GC_throw_bad_alloc`
# symbol is missing during linking.
, enableGC ? !stdenv.hostPlatform.isWindows
}:
let
inherit (lib) fileset;
in
mkMesonDerivation (finalAttrs: {
pname = "nix-expr";
inherit version;
workDir = ./.;
fileset = fileset.unions [
../../build-utils-meson
./build-utils-meson
../../.version
./.version
./meson.build
./meson.options
./primops/meson.build
(fileset.fileFilter (file: file.hasExt "cc") ./.)
(fileset.fileFilter (file: file.hasExt "hh") ./.)
./lexer.l
./parser.y
(fileset.fileFilter (file: file.hasExt "nix") ./.)
];
outputs = [ "out" "dev" ];
nativeBuildInputs = [
meson
ninja
pkg-config
bison
flex
cmake
];
buildInputs = [
toml11
];
propagatedBuildInputs = [
nix-util
nix-store
nix-fetchers
boost
nlohmann_json
] ++ lib.optional enableGC boehmgc;
preConfigure =
# "Inline" .version so it's not a symlink, and includes the suffix.
# Do the meson utils, without modification.
''
chmod u+w ./.version
echo ${version} > ../../.version
'';
mesonFlags = [
(lib.mesonEnable "gc" enableGC)
];
env = {
# Needed for Meson to find Boost.
# https://github.com/NixOS/nixpkgs/issues/86131.
BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
} // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
LDFLAGS = "-fuse-ld=gold";
};
separateDebugInfo = !stdenv.hostPlatform.isStatic;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
meta = {
platforms = lib.platforms.unix ++ lib.platforms.windows;
};
})

83
src/libexpr/parser-state.hh
View file

@ -1,6 +1,8 @@
#pragma once
///@file
#include <limits>
#include "eval.hh"
namespace nix {
@ -18,27 +20,62 @@ struct StringToken
operator std::string_view() const { return {p, l}; }
};
// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y.
struct ParserLocation
{
int first_line, first_column;
int last_line, last_column;
int beginOffset;
int endOffset;
// backup to recover from yyless(0)
int stashed_first_column, stashed_last_column;
int stashedBeginOffset, stashedEndOffset;
void stash() {
stashed_first_column = first_column;
stashed_last_column = last_column;
stashedBeginOffset = beginOffset;
stashedEndOffset = endOffset;
}
void unstash() {
first_column = stashed_first_column;
last_column = stashed_last_column;
beginOffset = stashedBeginOffset;
endOffset = stashedEndOffset;
}
/** Latest doc comment position, or 0. */
int doc_comment_first_column, doc_comment_last_column;
};
struct LexerState
{
/**
* Tracks the distance to the last doc comment, in terms of lexer tokens.
*
* The lexer sets this to 0 when reading a doc comment, and increments it
* for every matched rule; see `lexer-helpers.cc`.
* Whitespace and comment rules decrement the distance, so that they result
* in a net 0 change in distance.
*/
int docCommentDistance = std::numeric_limits<int>::max();
/**
* The location of the last doc comment.
*
* (stashing fields are not used)
*/
ParserLocation lastDocCommentLoc;
/**
* @brief Maps some positions to a DocComment, where the comment is relevant to the location.
*/
std::unordered_map<PosIdx, DocComment> & positionToDocComment;
PosTable & positions;
PosTable::Origin origin;
PosIdx at(const ParserLocation & loc);
};
struct ParserState
{
const LexerState & lexerState;
SymbolTable & symbols;
PosTable & positions;
Expr * result;
@ -46,10 +83,11 @@ struct ParserState
PosTable::Origin origin;
const ref<SourceAccessor> rootFS;
const Expr::AstSymbols & s;
const EvalSettings & settings;
void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos);
void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc);
Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
Expr * stripIndentation(const PosIdx pos,
std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
@ -73,11 +111,12 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre
});
}
inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos)
inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc)
{
AttrPath::iterator i;
// All attrpaths have at least one attr
assert(!attrPath.empty());
auto pos = at(loc);
// Checking attrPath validity.
// ===========================
for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
@ -142,6 +181,12 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
} else {
attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
}
auto it = lexerState.positionToDocComment.find(pos);
if (it != lexerState.positionToDocComment.end()) {
e->setDocComment(it->second);
lexerState.positionToDocComment.emplace(at(exprLoc), it->second);
}
}
inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
@ -254,12 +299,23 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
s2 = std::string(s2, 0, p + 1);
}
es2->emplace_back(i->first, new ExprString(std::move(s2)));
// Ignore empty strings for a minor optimisation and AST simplification
if (s2 != "") {
es2->emplace_back(i->first, new ExprString(std::move(s2)));
}
};
for (; i != es.end(); ++i, --n) {
std::visit(overloaded { trimExpr, trimString }, i->second);
}
// If there is nothing at all, return the empty string directly.
// This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters.
if (es2->size() == 0) {
auto *const result = new ExprString("");
delete es2;
return result;
}
/* If this is a single string, then don't do a concatenation. */
if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
auto *const result = (*es2)[0].second;
@ -269,9 +325,14 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
return new ExprConcatStrings(pos, true, es2);
}
inline PosIdx LexerState::at(const ParserLocation & loc)
{
return positions.add(origin, loc.beginOffset);
}
inline PosIdx ParserState::at(const ParserLocation & loc)
{
return positions.add(origin, loc.first_column);
return positions.add(origin, loc.beginOffset);
}
}

213
src/libexpr/parser.y
View file

@ -1,4 +1,4 @@
%glr-parser
%define api.location.type { ::nix::ParserLocation }
%define api.pure
%locations
%define parse.error verbose
@ -8,8 +8,7 @@
%parse-param { nix::ParserState * state }
%lex-param { void * scanner }
%lex-param { nix::ParserState * state }
%expect 1
%expect-rr 1
%expect 0
%code requires {
@ -25,22 +24,50 @@
#include "nixexpr.hh"
#include "eval.hh"
#include "eval-settings.hh"
#include "globals.hh"
#include "parser-state.hh"
#define YYLTYPE ::nix::ParserLocation
// Bison seems to have difficulty growing the parser stack when using C++ with
// a custom location type. This undocumented macro tells Bison that our
// location type is "trivially copyable" in C++-ese, so it is safe to use the
// same memcpy macro it uses to grow the stack that it uses with its own
// default location type. Without this, we get "error: memory exhausted" when
// parsing some large Nix files. Our other options are to increase the initial
// stack size (200 by default) to be as large as we ever want to support (so
// that growing the stack is unnecessary), or redefine the stack-relocation
// macro ourselves (which is also undocumented).
#define YYLTYPE_IS_TRIVIAL 1
#define YY_DECL int yylex \
(YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
// For efficiency, we only track offsets; not line,column coordinates
# define YYLLOC_DEFAULT(Current, Rhs, N) \
do \
if (N) \
{ \
(Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset; \
(Current).endOffset = YYRHSLOC (Rhs, N).endOffset; \
} \
else \
{ \
(Current).beginOffset = (Current).endOffset = \
YYRHSLOC (Rhs, 0).endOffset; \
} \
while (0)
namespace nix {
typedef std::unordered_map<PosIdx, DocComment> DocCommentMap;
Expr * parseExprFromBuf(
char * text,
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
SymbolTable & symbols,
const EvalSettings & settings,
PosTable & positions,
DocCommentMap & docComments,
const ref<SourceAccessor> rootFS,
const Expr::AstSymbols & astSymbols);
@ -59,14 +86,13 @@ YY_DECL;
using namespace nix;
#define CUR_POS state->at(*yylocp)
#define CUR_POS state->at(yyloc)
void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
{
if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
loc->first_column = loc->last_column;
loc->first_line = loc->last_line;
loc->beginOffset = loc->endOffset;
}
throw ParseError({
.msg = HintFmt(error),
@ -74,6 +100,22 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
});
}
#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos))
static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) {
auto it = lexerState.positionToDocComment.find(start);
if (it != lexerState.positionToDocComment.end()) {
lambda->setDocComment(it->second);
}
}
static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
e2->args.push_back(arg);
return fn;
}
return new ExprCall(pos, fn, {arg});
}
%}
@ -98,9 +140,10 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
%type <e> start expr expr_function expr_if expr_op
%type <e> expr_select expr_simple expr_app
%type <e> expr_pipe_from expr_pipe_into
%type <list> expr_list
%type <attrs> binds
%type <formals> formals
%type <attrs> binds binds1
%type <formals> formals formal_set
%type <formal> formal
%type <attrNames> attrpath
%type <inheritAttrs> attrs
@ -115,10 +158,12 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
%token <path> PATH HPATH SPATH PATH_END
%token <uri> URI
%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
%token PIPE_FROM PIPE_INTO /* <| and |> */
%token DOLLAR_CURLY /* == ${ */
%token IND_STRING_OPEN IND_STRING_CLOSE
%token ELLIPSIS
%right IMPL
%left OR
%left AND
@ -140,18 +185,28 @@ expr: expr_function;
expr_function
: ID ':' expr_function
{ $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); }
| '{' formals '}' ':' expr_function
{ $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); }
| '{' formals '}' '@' ID ':' expr_function
{
auto arg = state->symbols.create($5);
$$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
{ auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3);
$$ = me;
SET_DOC_POS(me, @1);
}
| ID '@' '{' formals '}' ':' expr_function
| formal_set ':' expr_function[body]
{ auto me = new ExprLambda(CUR_POS, state->validateFormals($formal_set), $body);
$$ = me;
SET_DOC_POS(me, @1);
}
| formal_set '@' ID ':' expr_function[body]
{
auto arg = state->symbols.create($1);
$$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
auto arg = state->symbols.create($ID);
auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body);
$$ = me;
SET_DOC_POS(me, @1);
}
| ID '@' formal_set ':' expr_function[body]
{
auto arg = state->symbols.create($ID);
auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body);
$$ = me;
SET_DOC_POS(me, @1);
}
| ASSERT expr ';' expr_function
{ $$ = new ExprAssert(CUR_POS, $2, $4); }
@ -170,9 +225,21 @@ expr_function
expr_if
: IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
| expr_pipe_from
| expr_pipe_into
| expr_op
;
expr_pipe_from
: expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); }
| expr_op PIPE_FROM expr_op { $$ = makeCall(state->at(@2), $1, $3); }
;
expr_pipe_into
: expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); }
| expr_op PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); }
;
expr_op
: '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
| '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); }
@ -197,13 +264,7 @@ expr_op
;
expr_app
: expr_app expr_select {
if (auto e2 = dynamic_cast<ExprCall *>($1)) {
e2->args.push_back($2);
$$ = $1;
} else
$$ = new ExprCall(CUR_POS, $1, {$2});
}
: expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); }
| expr_select
;
@ -259,11 +320,13 @@ expr_simple
/* Let expressions `let {..., body = ...}' are just desugared
into `(rec {..., body = ...}).body'. */
| LET '{' binds '}'
{ $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); }
{ $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); }
| REC '{' binds '}'
{ $3->recursive = true; $$ = $3; }
| '{' binds '}'
{ $$ = $2; }
{ $3->recursive = true; $3->pos = CUR_POS; $$ = $3; }
| '{' binds1 '}'
{ $2->pos = CUR_POS; $$ = $2; }
| '{' '}'
{ $$ = new ExprAttrs(CUR_POS); }
| '[' expr_list ']' { $$ = $2; }
;
@ -287,14 +350,14 @@ string_parts_interpolated
path_start
: PATH {
Path path(absPath({$1.p, $1.l}, state->basePath.path.abs()));
Path path(absPath(std::string_view{$1.p, $1.l}, state->basePath.path.abs()));
/* add back in the trailing '/' to the first segment */
if ($1.p[$1.l-1] == '/' && $1.l > 1)
path += "/";
$$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
}
| HPATH {
if (evalSettings.pureEval) {
if (state->settings.pureEval) {
throw Error(
"the path '%s' can not be resolved in pure mode",
std::string_view($1.p, $1.l)
@ -312,37 +375,50 @@ ind_string_parts
;
binds
: binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
| binds INHERIT attrs ';'
{ $$ = $1;
for (auto & [i, iPos] : *$3) {
if ($$->attrs.find(i.symbol) != $$->attrs.end())
state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
$$->attrs.emplace(
: binds1
| { $$ = new ExprAttrs; }
;
binds1
: binds1[accum] attrpath '=' expr ';'
{ $$ = $accum;
state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
delete $attrpath;
}
| binds[accum] INHERIT attrs ';'
{ $$ = $accum;
for (auto & [i, iPos] : *$attrs) {
if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs.emplace(
i.symbol,
ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
}
delete $3;
delete $attrs;
}
| binds INHERIT '(' expr ')' attrs ';'
{ $$ = $1;
if (!$$->inheritFromExprs)
$$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
$$->inheritFromExprs->push_back($4);
auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
for (auto & [i, iPos] : *$6) {
if ($$->attrs.find(i.symbol) != $$->attrs.end())
state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
$$->attrs.emplace(
| binds[accum] INHERIT '(' expr ')' attrs ';'
{ $$ = $accum;
if (!$accum->inheritFromExprs)
$accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
$accum->inheritFromExprs->push_back($expr);
auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1);
for (auto & [i, iPos] : *$attrs) {
if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs.emplace(
i.symbol,
ExprAttrs::AttrDef(
new ExprSelect(iPos, from, i.symbol),
iPos,
ExprAttrs::AttrDef::Kind::InheritedFrom));
}
delete $6;
delete $attrs;
}
| attrpath '=' expr ';'
{ $$ = new ExprAttrs;
state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
delete $attrpath;
}
| { $$ = new ExprAttrs(state->at(@0)); }
;
attrs
@ -400,15 +476,19 @@ expr_list
| { $$ = new ExprList; }
;
formal_set
: '{' formals ',' ELLIPSIS '}' { $$ = $formals; $$->ellipsis = true; }
| '{' ELLIPSIS '}' { $$ = new Formals; $$->ellipsis = true; }
| '{' formals ',' '}' { $$ = $formals; $$->ellipsis = false; }
| '{' formals '}' { $$ = $formals; $$->ellipsis = false; }
| '{' '}' { $$ = new Formals; $$->ellipsis = false; }
;
formals
: formal ',' formals
{ $$ = $3; $$->formals.emplace_back(*$1); delete $1; }
: formals[accum] ',' formal
{ $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; }
| formal
{ $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; }
|
{ $$ = new Formals; $$->ellipsis = false; }
| ELLIPSIS
{ $$ = new Formals; $$->ellipsis = true; }
{ $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; }
;
formal
@ -429,21 +509,30 @@ Expr * parseExprFromBuf(
Pos::Origin origin,
const SourcePath & basePath,
SymbolTable & symbols,
const EvalSettings & settings,
PosTable & positions,
DocCommentMap & docComments,
const ref<SourceAccessor> rootFS,
const Expr::AstSymbols & astSymbols)
{
yyscan_t scanner;
LexerState lexerState {
.positionToDocComment = docComments,
.positions = positions,
.origin = positions.addOrigin(origin, length),
};
ParserState state {
.lexerState = lexerState,
.symbols = symbols,
.positions = positions,
.basePath = basePath,
.origin = positions.addOrigin(origin, length),
.origin = lexerState.origin,
.rootFS = rootFS,
.s = astSymbols,
.settings = settings,
};
yylex_init(&scanner);
yylex_init_extra(&lexerState, &scanner);
Finally _destroy([&] { yylex_destroy(scanner); });
yy_scan_buffer(text, length, scanner);

23
src/libexpr/pos-idx.hh
View file

@ -1,6 +1,7 @@
#pragma once
#include <cinttypes>
#include <functional>
namespace nix {
@ -8,6 +9,7 @@ class PosIdx
{
friend struct LazyPosAcessors;
friend class PosTable;
friend class std::hash<PosIdx>;
private:
uint32_t id;
@ -28,9 +30,9 @@ public:
return id > 0;
}
bool operator<(const PosIdx other) const
auto operator<=>(const PosIdx other) const
{
return id < other.id;
return id <=> other.id;
}
bool operator==(const PosIdx other) const
@ -38,12 +40,25 @@ public:
return id == other.id;
}
bool operator!=(const PosIdx other) const
size_t hash() const noexcept
{
return id != other.id;
return std::hash<uint32_t>{}(id);
}
};
inline PosIdx noPos = {};
}
namespace std {
template<>
struct hash<nix::PosIdx>
{
std::size_t operator()(nix::PosIdx pos) const noexcept
{
return pos.hash();
}
};
} // namespace std

4
src/libexpr/pos-table.hh
View file

@ -1,10 +1,8 @@
#pragma once
#include <cinttypes>
#include <numeric>
#include <cstdint>
#include <vector>
#include "chunked-vector.hh"
#include "pos-idx.hh"
#include "position.hh"
#include "sync.hh"

466
src/libexpr/primops.cc
View file

@ -1,11 +1,9 @@
#include "archive.hh"
#include "derivations.hh"
#include "downstream-placeholder.hh"
#include "eval-inline.hh"
#include "eval.hh"
#include "eval-settings.hh"
#include "gc-small-vector.hh"
#include "globals.hh"
#include "json-to-value.hh"
#include "names.hh"
#include "path-references.hh"
@ -26,6 +24,7 @@
#include <algorithm>
#include <cstring>
#include <sstream>
#include <regex>
#ifndef _WIN32
@ -77,8 +76,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
if (drvs.empty()) return {};
if (isIFD && !evalSettings.enableImportFromDerivation)
error<EvalError>(
if (isIFD && !settings.enableImportFromDerivation)
error<EvalBaseError>(
"cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
drvs.begin()->to_string(*store)
).debugThrow();
@ -427,7 +426,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Val
t = args[0]->external()->typeOf();
break;
case nFloat: t = "float"; break;
case nThunk: abort();
case nThunk: unreachable();
}
v.mkString(t);
}
@ -588,9 +587,9 @@ struct CompareValues
{
try {
if (v1->type() == nFloat && v2->type() == nInt)
return v1->fpoint() < v2->integer();
return v1->fpoint() < v2->integer().value;
if (v1->type() == nInt && v2->type() == nFloat)
return v1->integer() < v2->fpoint();
return v1->integer().value < v2->fpoint();
if (v1->type() != v2->type())
state.error<EvalError>("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow();
// Allow selecting a subset of enum values
@ -720,7 +719,7 @@ static RegisterPrimOp primop_genericClosure(PrimOp {
.doc = R"(
`builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function.
It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attrbute sets:
It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attribute sets:
- `startSet`:
The initial list of attribute sets.
@ -732,11 +731,12 @@ static RegisterPrimOp primop_genericClosure(PrimOp {
Each attribute set in the list `startSet` and the list returned by `operator` must have an attribute `key`, which must support equality comparison.
The value of `key` can be one of the following types:
- [Number](@docroot@/language/values.md#type-number)
- [Boolean](@docroot@/language/values.md#type-boolean)
- [String](@docroot@/language/values.md#type-string)
- [Path](@docroot@/language/values.md#type-path)
- [List](@docroot@/language/values.md#list)
- [Int](@docroot@/language/types.md#type-int)
- [Float](@docroot@/language/types.md#type-float)
- [Boolean](@docroot@/language/types.md#type-boolean)
- [String](@docroot@/language/types.md#type-string)
- [Path](@docroot@/language/types.md#type-path)
- [List](@docroot@/language/types.md#list)
The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added.
Items are compared by their `key` attribute.
@ -779,15 +779,14 @@ static RegisterPrimOp primop_break({
)",
.fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
if (state.debugRepl && !state.debugTraces.empty()) {
if (state.canDebug()) {
auto error = Error(ErrorInfo {
.level = lvlInfo,
.msg = HintFmt("breakpoint reached"),
.pos = state.positions[pos],
});
auto & dt = state.debugTraces.front();
state.runDebugRepl(&error, dt.env, dt.expr);
state.runDebugRepl(&error);
}
// Return the value we were passed.
@ -806,7 +805,7 @@ static RegisterPrimOp primop_abort({
NixStringContext context;
auto s = state.coerceToString(pos, *args[0], context,
"while evaluating the error message passed to builtins.abort").toOwned();
state.error<Abort>("evaluation aborted with the following error message: '%1%'", s).debugThrow();
state.error<Abort>("evaluation aborted with the following error message: '%1%'", s).setIsFromExpr().debugThrow();
}
});
@ -825,7 +824,7 @@ static RegisterPrimOp primop_throw({
NixStringContext context;
auto s = state.coerceToString(pos, *args[0], context,
"while evaluating the error message passed to builtin.throw").toOwned();
state.error<ThrownError>(s).debugThrow();
state.error<ThrownError>(s).setIsFromExpr().debugThrow();
}
});
@ -901,7 +900,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va
MaintainCount trylevel(state.trylevel);
ReplExitStatus (* savedDebugRepl)(ref<EvalState> es, const ValMap & extraEnv) = nullptr;
if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry)
if (state.debugRepl && state.settings.ignoreExceptionsDuringTry)
{
/* to prevent starting the repl from exceptions withing a tryEval, null it. */
savedDebugRepl = state.debugRepl;
@ -950,7 +949,7 @@ static RegisterPrimOp primop_tryEval({
static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv"));
v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or(""));
v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""));
}
static RegisterPrimOp primop_getEnv({
@ -1017,9 +1016,8 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu
printError("trace: %1%", args[0]->string_view());
else
printError("trace: %1%", ValuePrinter(state, *args[0]));
if (evalSettings.builtinsTraceDebugger && state.debugRepl && !state.debugTraces.empty()) {
const DebugTrace & last = state.debugTraces.front();
state.runDebugRepl(nullptr, last.env, last.expr);
if (state.settings.builtinsTraceDebugger) {
state.runDebugRepl(nullptr);
}
state.forceValue(*args[1], pos);
v = *args[1];
@ -1042,6 +1040,55 @@ static RegisterPrimOp primop_trace({
.fun = prim_trace,
});
static void prim_warn(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
// We only accept a string argument for now. The use case for pretty printing a value is covered by `trace`.
// By rejecting non-strings we allow future versions to add more features without breaking existing code.
auto msgStr = state.forceString(*args[0], pos, "while evaluating the first argument; the message passed to builtins.warn");
{
BaseError msg(std::string{msgStr});
msg.atPos(state.positions[pos]);
auto info = msg.info();
info.level = lvlWarn;
info.isFromExpr = true;
logWarning(info);
}
if (state.settings.builtinsAbortOnWarn) {
// Not an EvalError or subclass, which would cause the error to be stored in the eval cache.
state.error<EvalBaseError>("aborting to reveal stack trace of warning, as abort-on-warn is set").setIsFromExpr().debugThrow();
}
if (state.settings.builtinsTraceDebugger || state.settings.builtinsDebuggerOnWarn) {
state.runDebugRepl(nullptr);
}
state.forceValue(*args[1], pos);
v = *args[1];
}
static RegisterPrimOp primop_warn({
.name = "__warn",
.args = {"e1", "e2"},
.doc = R"(
Evaluate *e1*, which must be a string and print iton standard error as a warning.
Then return *e2*.
This function is useful for non-critical situations where attention is advisable.
If the
[`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace)
or [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn)
option is set to `true` and the `--debugger` flag is given, the
interactive debugger will be started when `warn` is called (like
[`break`](@docroot@/language/builtins.md#builtins-break)).
If the
[`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn)
option is set, the evaluation will be aborted after the warning is printed.
This is useful to reveal the stack trace of the warning, when the context is non-interactive and a debugger can not be launched.
)",
.fun = prim_warn,
});
/* Takes two arguments and evaluates to the second one. Used as the
* builtins.traceVerbose implementation when --trace-verbose is not enabled
@ -1115,12 +1162,34 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
}
}
/**
* Early validation for the derivation name, for better error message.
* It is checked again when constructing store paths.
*
* @todo Check that the `.drv` suffix also fits.
*/
static void checkDerivationName(EvalState & state, std::string_view drvName)
{
try {
checkName(drvName);
} catch (BadStorePathName & e) {
// "Please pass a different name": Users may not be aware that they can
// pass a different one, in functions like `fetchurl` where the name
// is optional.
// Note that Nixpkgs generally won't trigger this, because `mkDerivation`
// sanitizes the name.
state.error<EvalError>("invalid derivation name: %s. Please pass a different '%s'.", Uncolored(e.message()), "name").debugThrow();
}
}
static void derivationStrictInternal(
EvalState & state,
const std::string & drvName,
const Bindings * attrs,
Value & v)
{
checkDerivationName(state, drvName);
/* Check whether attributes should be passed as a JSON file. */
using nlohmann::json;
std::optional<json> jsonObject;
@ -1155,13 +1224,13 @@ static void derivationStrictInternal(
for (auto & i : attrs->lexicographicOrder(state.symbols)) {
if (i->name == state.sIgnoreNulls) continue;
const std::string & key = state.symbols[i->name];
auto key = state.symbols[i->name];
vomit("processing attribute '%1%'", key);
auto handleHashMode = [&](const std::string_view s) {
if (s == "recursive") {
// back compat, new name is "nar"
ingestionMethod = FileIngestionMethod::Recursive;
ingestionMethod = ContentAddressMethod::Raw::NixArchive;
} else try {
ingestionMethod = ContentAddressMethod::parse(s);
} catch (UsageError &) {
@ -1169,9 +1238,9 @@ static void derivationStrictInternal(
"invalid value '%s' for 'outputHashMode' attribute", s
).atPos(v).debugThrow();
}
if (ingestionMethod == TextIngestionMethod {})
if (ingestionMethod == ContentAddressMethod::Raw::Text)
experimentalFeatureSettings.require(Xp::DynamicDerivations);
if (ingestionMethod == FileIngestionMethod::Git)
if (ingestionMethod == ContentAddressMethod::Raw::Git)
experimentalFeatureSettings.require(Xp::GitHashing);
};
@ -1239,7 +1308,7 @@ static void derivationStrictInternal(
if (i->name == state.sStructuredAttrs) continue;
(*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context);
jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context));
if (i->name == state.sBuilder)
drv.builder = state.forceString(*i->value, context, pos, context_below);
@ -1260,6 +1329,20 @@ static void derivationStrictInternal(
handleOutputs(ss);
}
if (i->name == state.sAllowedReferences)
warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks.<output>.allowedReferences' instead", drvName);
if (i->name == state.sAllowedRequisites)
warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks.<output>.allowedRequisites' instead", drvName);
if (i->name == state.sDisallowedReferences)
warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks.<output>.disallowedReferences' instead", drvName);
if (i->name == state.sDisallowedRequisites)
warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks.<output>.disallowedRequisites' instead", drvName);
if (i->name == state.sMaxSize)
warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks.<output>.maxSize' instead", drvName);
if (i->name == state.sMaxClosureSize)
warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks.<output>.maxClosureSize' instead", drvName);
} else {
auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned();
drv.env.emplace(key, s);
@ -1329,7 +1412,7 @@ static void derivationStrictInternal(
/* Check whether the derivation name is valid. */
if (isDerivation(drvName) &&
!(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } &&
!(ingestionMethod == ContentAddressMethod::Raw::Text &&
outputs.size() == 1 &&
*(outputs.begin()) == "out"))
{
@ -1351,7 +1434,7 @@ static void derivationStrictInternal(
auto h = newHashAllowEmpty(*outputHash, outputHashAlgo);
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::Flat);
DerivationOutput::CAFixed dof {
.ca = ContentAddress {
@ -1370,7 +1453,7 @@ static void derivationStrictInternal(
.atPos(v).debugThrow();
auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256);
auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive);
for (auto & i : outputs) {
drv.env[i] = hashPlaceholder(i);
@ -1516,7 +1599,7 @@ static RegisterPrimOp primop_toPath({
corner cases. */
static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
if (evalSettings.pureEval)
if (state.settings.pureEval)
state.error<EvalError>(
"'%s' is not allowed in pure evaluation mode",
"builtins.storePath"
@ -1626,7 +1709,7 @@ static RegisterPrimOp primop_baseNameOf({
.name = "baseNameOf",
.args = {"x"},
.doc = R"(
Return the *base name* of either a [path value](@docroot@/language/values.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules.
Return the *base name* of either a [path value](@docroot@/language/types.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules.
For a path value, the *base name* is considered to be the part of the path after the last directory separator, including any file extensions.
This is the simple case, as path values don't have trailing slashes.
@ -1760,35 +1843,7 @@ static RegisterPrimOp primop_findFile(PrimOp {
.doc = R"(
Find *lookup-path* in *search-path*.
A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes:
- `prefix` is a relative path.
- `path` denotes a file system location
The exact syntax depends on the command line interface.
Examples of search path attribute sets:
- ```
{
prefix = "nixos-config";
path = "/etc/nixos/configuration.nix";
}
```
- ```
{
prefix = "";
path = "/nix/var/nix/profiles/per-user/root/channels";
}
```
The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match:
- If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`.
Note that the `path` may need to be downloaded at this point to look inside.
- If the suffix is found inside that directory, then the entry is a match.
The combined absolute path of the directory (now downloaded if need be) and the suffix is returned.
[Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath):
[Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath):
```nix
<nixpkgs>
@ -1799,6 +1854,119 @@ static RegisterPrimOp primop_findFile(PrimOp {
```nix
builtins.findFile builtins.nixPath "nixpkgs"
```
A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes:
- `prefix` is a relative path.
- `path` denotes a file system location
Examples of search path attribute sets:
- ```
{
prefix = "";
path = "/nix/var/nix/profiles/per-user/root/channels";
}
```
- ```
{
prefix = "nixos-config";
path = "/etc/nixos/configuration.nix";
}
```
- ```
{
prefix = "nixpkgs";
path = "https://github.com/NixOS/nixpkgs/tarballs/master";
}
```
- ```
{
prefix = "nixpkgs";
path = "channel:nixpkgs-unstable";
}
```
- ```
{
prefix = "flake-compat";
path = "flake:github:edolstra/flake-compat";
}
```
The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match:
- If a prefix of `lookup-path` matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`.
The contents of `path` may need to be downloaded at this point to look inside.
- If the suffix is found inside that directory, then the entry is a match.
The combined absolute path of the directory (now downloaded if need be) and the suffix is returned.
> **Example**
>
> A *search-path* value
>
> ```
> [
> {
> prefix = "";
> path = "/home/eelco/Dev";
> }
> {
> prefix = "nixos-config";
> path = "/etc/nixos";
> }
> ]
> ```
>
> and a *lookup-path* value `"nixos-config"` will cause Nix to try `/home/eelco/Dev/nixos-config` and `/etc/nixos` in that order and return the first path that exists.
If `path` starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location.
The tarball must consist of a single top-level directory.
The URLs of the tarballs from the official `nixos.org` channels can be abbreviated as `channel:<channel-name>`.
See [documentation on `nix-channel`](@docroot@/command-ref/nix-channel.md) for details about channels.
> **Example**
>
> These two search path entries are equivalent:
>
> - ```
> {
> prefix = "nixpkgs";
> path = "channel:nixpkgs-unstable";
> }
> ```
> - ```
> {
> prefix = "nixpkgs";
> path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz";
> }
> ```
Search paths can also point to source trees using [flake URLs](@docroot@/command-ref/new-cli/nix3-flake.md#url-like-syntax).
> **Example**
>
> The search path entry
>
> ```
> {
> prefix = "nixpkgs";
> path = "flake:nixpkgs";
> }
> ```
> specifies that the prefix `nixpkgs` shall refer to the source tree downloaded from the `nixpkgs` entry in the flake registry.
>
> Similarly
>
> ```
> {
> prefix = "nixpkgs";
> path = "flake:github:nixos/nixpkgs/nixos-22.05";
> }
> ```
>
> makes `<nixpkgs>` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub.
)",
.fun = prim_findFile,
});
@ -2146,7 +2314,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
})
: ({
StringSource s { contents };
state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair);
});
/* Note: we don't need to add `context' to the context of the
@ -2209,7 +2377,7 @@ static RegisterPrimOp primop_toFile({
```
Note that `${configFile}` is a
[string interpolation](@docroot@/language/values.md#type-string), so the result of the
[string interpolation](@docroot@/language/types.md#type-string), so the result of the
expression `configFile`
(i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be
spliced into the resulting string.
@ -2261,7 +2429,7 @@ static void addPath(
std::string_view name,
SourcePath path,
Value * filterFun,
FileIngestionMethod method,
ContentAddressMethod method,
const std::optional<Hash> expectedHash,
Value & v,
const NixStringContext & context)
@ -2293,11 +2461,10 @@ static void addPath(
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
.method = method,
.hash = *expectedHash,
.references = {},
});
expectedStorePath = state.store->makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(
method,
*expectedHash,
{}));
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
auto dstPath = fetchToStore(
@ -2330,7 +2497,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
"while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context);
}
static RegisterPrimOp primop_filterSource({
@ -2393,7 +2560,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
std::optional<SourcePath> path;
std::string name;
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
auto method = ContentAddressMethod::Raw::NixArchive;
std::optional<Hash> expectedHash;
NixStringContext context;
@ -2408,7 +2575,9 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
else if (n == "filter")
state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path");
else if (n == "recursive")
method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") };
method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path")
? ContentAddressMethod::Raw::NixArchive
: ContentAddressMethod::Raw::Flat;
else if (n == "sha256")
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
else
@ -2593,13 +2762,13 @@ static struct LazyPosAcessors {
PrimOp primop_lineOfPos{
.arity = 1,
.fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) {
v.mkInt(state.positions[PosIdx(args[0]->integer())].line);
v.mkInt(state.positions[PosIdx(args[0]->integer().value)].line);
}
};
PrimOp primop_columnOfPos{
.arity = 1,
.fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) {
v.mkInt(state.positions[PosIdx(args[0]->integer())].column);
v.mkInt(state.positions[PosIdx(args[0]->integer().value)].column);
}
};
@ -3075,7 +3244,8 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val
/* Return the n-1'th element of a list. */
static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
elemAt(state, pos, *args[0], state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt"), v);
NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value;
elemAt(state, pos, *args[0], elem, v);
}
static RegisterPrimOp primop_elemAt({
@ -3369,10 +3539,12 @@ static RegisterPrimOp primop_all({
static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList");
auto len_ = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList").value;
if (len < 0)
state.error<EvalError>("cannot create list of size %1%", len).atPos(pos).debugThrow();
if (len_ < 0)
state.error<EvalError>("cannot create list of size %1%", len_).atPos(pos).debugThrow();
size_t len = size_t(len_);
// More strict than striclty (!) necessary, but acceptable
// as evaluating map without accessing any values makes little sense.
@ -3629,9 +3801,17 @@ static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value
if (args[0]->type() == nFloat || args[1]->type() == nFloat)
v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition")
+ state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition"));
else
v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the addition")
+ state.forceInt(*args[1], pos, "while evaluating the second argument of the addition"));
else {
auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the addition");
auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the addition");
auto result_ = i1 + i2;
if (auto result = result_.valueChecked(); result.has_value()) {
v.mkInt(*result);
} else {
state.error<EvalError>("integer overflow in adding %1% + %2%", i1, i2).atPos(pos).debugThrow();
}
}
}
static RegisterPrimOp primop_add({
@ -3650,9 +3830,18 @@ static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value
if (args[0]->type() == nFloat || args[1]->type() == nFloat)
v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction")
- state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction"));
else
v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction")
- state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction"));
else {
auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction");
auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction");
auto result_ = i1 - i2;
if (auto result = result_.valueChecked(); result.has_value()) {
v.mkInt(*result);
} else {
state.error<EvalError>("integer overflow in subtracting %1% - %2%", i1, i2).atPos(pos).debugThrow();
}
}
}
static RegisterPrimOp primop_sub({
@ -3671,9 +3860,18 @@ static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value
if (args[0]->type() == nFloat || args[1]->type() == nFloat)
v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication")
* state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication"));
else
v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication")
* state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication"));
else {
auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication");
auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication");
auto result_ = i1 * i2;
if (auto result = result_.valueChecked(); result.has_value()) {
v.mkInt(*result);
} else {
state.error<EvalError>("integer overflow in multiplying %1% * %2%", i1, i2).atPos(pos).debugThrow();
}
}
}
static RegisterPrimOp primop_mul({
@ -3700,10 +3898,12 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value
NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division");
NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division");
/* Avoid division overflow as it might raise SIGFPE. */
if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
state.error<EvalError>("overflow in integer division").atPos(pos).debugThrow();
v.mkInt(i1 / i2);
auto result_ = i1 / i2;
if (auto result = result_.valueChecked(); result.has_value()) {
v.mkInt(*result);
} else {
state.error<EvalError>("integer overflow in dividing %1% / %2%", i1, i2).atPos(pos).debugThrow();
}
}
}
@ -3718,8 +3918,9 @@ static RegisterPrimOp primop_div({
static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd")
& state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd"));
auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd");
auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd");
v.mkInt(i1.value & i2.value);
}
static RegisterPrimOp primop_bitAnd({
@ -3733,8 +3934,10 @@ static RegisterPrimOp primop_bitAnd({
static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr")
| state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr"));
auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr");
auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr");
v.mkInt(i1.value | i2.value);
}
static RegisterPrimOp primop_bitOr({
@ -3748,8 +3951,10 @@ static RegisterPrimOp primop_bitOr({
static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor")
^ state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor"));
auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor");
auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor");
v.mkInt(i1.value ^ i2.value);
}
static RegisterPrimOp primop_bitXor({
@ -3829,13 +4034,19 @@ static RegisterPrimOp primop_toString({
non-negative. */
static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring");
NixInt::Inner start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring").value;
if (start < 0)
state.error<EvalError>("negative start position in 'substring'").atPos(pos).debugThrow();
int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring");
NixInt::Inner len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring").value;
// Negative length may be idiomatically passed to builtins.substring to get
// the tail of the string.
if (len < 0) {
len = std::numeric_limits<NixInt::Inner>::max();
}
// Special-case on empty substring to avoid O(n) strlen
// This allows for the use of empty substrings to efficently capture string context
@ -3878,7 +4089,7 @@ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * arg
{
NixStringContext context;
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength");
v.mkInt(s->size());
v.mkInt(NixInt::Inner(s->size()));
}
static RegisterPrimOp primop_stringLength({
@ -4013,17 +4224,23 @@ static RegisterPrimOp primop_convertHash({
struct RegexCache
{
// TODO use C++20 transparent comparison when available
std::unordered_map<std::string_view, std::regex> cache;
std::list<std::string> keys;
struct State
{
// TODO use C++20 transparent comparison when available
std::unordered_map<std::string_view, std::regex> cache;
std::list<std::string> keys;
};
Sync<State> state_;
std::regex get(std::string_view re)
{
auto it = cache.find(re);
if (it != cache.end())
auto state(state_.lock());
auto it = state->cache.find(re);
if (it != state->cache.end())
return it->second;
keys.emplace_back(re);
return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second;
state->keys.emplace_back(re);
return state->cache.emplace(state->keys.back(), std::regex(state->keys.back(), std::regex::extended)).first->second;
}
};
@ -4356,7 +4573,8 @@ static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * *
{
auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions");
auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions");
v.mkInt(compareVersions(version1, version2));
auto result = compareVersions(version1, version2);
v.mkInt(result < 0 ? -1 : result > 0 ? 1 : 0);
}
static RegisterPrimOp primop_compareVersions({
@ -4428,7 +4646,7 @@ void EvalState::createBaseEnv()
addConstant("builtins", v, {
.type = nAttrs,
.doc = R"(
Contains all the [built-in functions](@docroot@/language/builtins.md) and values.
Contains all the built-in functions and values.
Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations:
@ -4448,7 +4666,7 @@ void EvalState::createBaseEnv()
It can be returned by
[comparison operators](@docroot@/language/operators.md#Comparison)
and used in
[conditional expressions](@docroot@/language/constructs.md#Conditionals).
[conditional expressions](@docroot@/language/syntax.md#Conditionals).
The name `true` is not special, and can be shadowed:
@ -4468,7 +4686,7 @@ void EvalState::createBaseEnv()
It can be returned by
[comparison operators](@docroot@/language/operators.md#Comparison)
and used in
[conditional expressions](@docroot@/language/constructs.md#Conditionals).
[conditional expressions](@docroot@/language/syntax.md#Conditionals).
The name `false` is not special, and can be shadowed:
@ -4493,7 +4711,7 @@ void EvalState::createBaseEnv()
)",
});
if (!evalSettings.pureEval) {
if (!settings.pureEval) {
v.mkInt(time(0));
}
addConstant("__currentTime", v, {
@ -4515,13 +4733,13 @@ void EvalState::createBaseEnv()
1683705525
```
The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
The [store path](@docroot@/store/store-path.md) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
)",
.impureOnly = true,
});
if (!evalSettings.pureEval)
v.mkString(evalSettings.getCurrentSystem());
if (!settings.pureEval)
v.mkString(settings.getCurrentSystem());
addConstant("__currentSystem", v, {
.type = nString,
.doc = R"(
@ -4601,7 +4819,7 @@ void EvalState::createBaseEnv()
#ifndef _WIN32 // TODO implement on Windows
// Miscellaneous
if (evalSettings.enableNativeCode) {
if (settings.enableNativeCode) {
addPrimOp({
.name = "__importNative",
.arity = 2,
@ -4624,7 +4842,7 @@ void EvalState::createBaseEnv()
error if `--trace-verbose` is enabled. Then return *e2*. This function
is useful for debugging.
)",
.fun = evalSettings.traceVerbose ? prim_trace : prim_second,
.fun = settings.traceVerbose ? prim_trace : prim_second,
});
/* Add a value containing the current Nix expression search path. */
@ -4639,7 +4857,17 @@ void EvalState::createBaseEnv()
addConstant("__nixPath", v, {
.type = nList,
.doc = R"(
The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md).
A list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md).
Its value is primarily determined by the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path), which are
- Overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment variable or the `--nix-path` option
- Extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`
> **Example**
>
> ```bash
> $ NIX_PATH= nix-instantiate --eval --expr "builtins.nixPath" -I foo=bar --no-pure-eval
> [ { path = "bar"; prefix = "foo"; } ]
> ```
Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and
[`builtins.findFile`](./builtins.html#builtins-findFile):

29
src/libexpr/primops/derivation.nix
View file

@ -1,6 +1,31 @@
/* This is the implementation of the ‘derivation’ builtin function.
It's actually a wrapper around the ‘derivationStrict’ primop. */
# This is the implementation of the ‘derivation’ builtin function.
# It's actually a wrapper around the ‘derivationStrict’ primop.
# Note that the following comment will be shown in :doc in the repl, but not in the manual.
/**
Create a derivation.
# Inputs
The single argument is an attribute set that describes what to build and how to build it.
See https://nix.dev/manual/nix/2.23/language/derivations
# Output
The result is an attribute set that describes the derivation.
Notably it contains the outputs, which in the context of the Nix language are special strings that refer to the output paths, which may not yet exist.
The realisation of these outputs only occurs when needed; for example
* When `nix-build` or a similar command is run, it realises the outputs that were requested on its command line.
See https://nix.dev/manual/nix/2.23/command-ref/nix-build
* When `import`, `readFile`, `readDir` or some other functions are called, they have to realise the outputs they depend on.
This is referred to as "import from derivation".
See https://nix.dev/manual/nix/2.23/language/import-from-derivation
Note that `derivation` is very bare-bones, and provides almost no commands during the build.
Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment.
*/
drvAttrs @ { outputs ? [ "out" ], ... }:
let

4
src/libexpr/primops/fetchMercurial.cc
View file

@ -53,7 +53,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
// whitelist. Ah well.
state.checkURI(url);
if (evalSettings.pureEval && !rev)
if (state.settings.pureEval && !rev)
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
fetchers::Attrs attrs;
@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
attrs.insert_or_assign("name", std::string(name));
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::Input::fromAttrs(std::move(attrs));
auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
auto [storePath, input2] = input.fetchToStore(state.store);

123
src/libexpr/primops/fetchTree.cc
View file

@ -1,4 +1,4 @@
#include "libfetchers/attrs.hh"
#include "attrs.hh"
#include "primops.hh"
#include "eval-inline.hh"
#include "eval-settings.hh"
@ -84,7 +84,7 @@ static void fetchTree(
Value & v,
const FetchTreeParams & params = FetchTreeParams{}
) {
fetchers::Input input;
fetchers::Input input { state.fetchSettings };
NixStringContext context;
std::optional<std::string> type;
if (params.isFetchGit) type = "git";
@ -121,9 +121,15 @@ static void fetchTree(
}
else if (attr.value->type() == nBool)
attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean()});
else if (attr.value->type() == nInt)
attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer()));
else if (state.symbols[attr.name] == "publicKeys") {
else if (attr.value->type() == nInt) {
auto intValue = attr.value->integer().value;
if (intValue < 0) {
state.error<EvalError>("negative value given for fetchTree attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow();
}
attrs.emplace(state.symbols[attr.name], uint64_t(intValue));
} else if (state.symbols[attr.name] == "publicKeys") {
experimentalFeatureSettings.require(Xp::VerifiedFetches);
attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump());
}
@ -136,13 +142,18 @@ static void fetchTree(
attrs.emplace("exportIgnore", Explicit<bool>{true});
}
// fetchTree should fetch git repos with shallow = true by default
if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
attrs.emplace("shallow", Explicit<bool>{true});
}
if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
state.error<EvalError>(
"attribute 'name' isn’t supported in call to 'fetchTree'"
).atPos(pos).debugThrow();
input = fetchers::Input::fromAttrs(std::move(attrs));
input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
} else {
auto url = state.coerceToString(pos, *args[0], context,
"while evaluating the first argument passed to the fetcher",
@ -155,20 +166,20 @@ static void fetchTree(
if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
attrs.emplace("exportIgnore", Explicit<bool>{true});
}
input = fetchers::Input::fromAttrs(std::move(attrs));
input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
} else {
if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
state.error<EvalError>(
"passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"
).atPos(pos).debugThrow();
input = fetchers::Input::fromURL(url);
input = fetchers::Input::fromURL(state.fetchSettings, url);
}
}
if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
input = lookupInRegistries(state.store, input).first;
if (evalSettings.pureEval && !input.isLocked()) {
if (state.settings.pureEval && !input.isLocked()) {
auto fetcher = "fetchTree";
if (params.isFetchGit)
fetcher = "fetchGit";
@ -199,8 +210,8 @@ static RegisterPrimOp primop_fetchTree({
.doc = R"(
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
- the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
- the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
- the resulting fixed-output [store path](@docroot@/store/store-path.md)
- the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
*input* must be an attribute set with the following attributes:
@ -319,6 +330,8 @@ static RegisterPrimOp primop_fetchTree({
- `ref` (String, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
@ -332,8 +345,9 @@ static RegisterPrimOp primop_fetchTree({
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `false`
Default: `true`
- `submodules` (Bool, optional)
@ -343,8 +357,11 @@ static RegisterPrimOp primop_fetchTree({
- `allRefs` (Bool, optional)
If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache.
Otherwise, only the latest commit is fetched if it is not already cached.
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
@ -371,7 +388,7 @@ static RegisterPrimOp primop_fetchTree({
- `"mercurial"`
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
> **Example**
>
@ -419,7 +436,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
state.forceValue(*args[0], pos);
if (args[0]->type() == nAttrs) {
bool isArgAttrs = args[0]->type() == nAttrs;
bool nameAttrPassed = false;
if (isArgAttrs) {
for (auto & attr : *args[0]->attrs()) {
std::string_view n(state.symbols[attr.name]);
@ -427,8 +447,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch");
else if (n == "sha256")
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256);
else if (n == "name")
else if (n == "name") {
nameAttrPassed = true;
name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch");
}
else
state.error<EvalError>("unsupported argument '%s' to '%s'", n, who)
.atPos(pos).debugThrow();
@ -441,14 +463,27 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch");
if (who == "fetchTarball")
url = evalSettings.resolvePseudoUrl(*url);
url = state.settings.resolvePseudoUrl(*url);
state.checkURI(*url);
if (name == "")
name = baseNameOf(*url);
if (evalSettings.pureEval && !expectedHash)
try {
checkName(name);
} catch (BadStorePathName & e) {
auto resolution =
nameAttrPassed ? HintFmt("Please change the value for the 'name' attribute passed to '%s', so that it can create a valid store path.", who) :
isArgAttrs ? HintFmt("Please add a valid 'name' attribute to the argument for '%s', so that it can create a valid store path.", who) :
HintFmt("Please pass an attribute set with 'url' and 'name' attributes to '%s', so that it can create a valid store path.", who);
state.error<EvalError>(
std::string("invalid store path name when fetching URL '%s': %s. %s"), *url, Uncolored(e.message()), Uncolored(resolution.str()))
.atPos(pos).debugThrow();
}
if (state.settings.pureEval && !expectedHash)
state.error<EvalError>("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow();
// early exit if pinned and already in the store
@ -456,7 +491,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
auto expectedPath = state.store->makeFixedOutputPath(
name,
FixedOutputInfo {
.method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
.method = unpack ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat,
.hash = *expectedHash,
.references = {}
});
@ -471,7 +506,11 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
// https://github.com/NixOS/nix/issues/4313
auto storePath =
unpack
? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name)
? fetchToStore(
*state.store,
fetchers::downloadTarball(state.store, state.fetchSettings, *url),
FetchMode::Copy,
name)
: fetchers::downloadFile(state.store, *url, name).storePath;
if (expectedHash) {
@ -499,9 +538,19 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, V
static RegisterPrimOp primop_fetchurl({
.name = "__fetchurl",
.args = {"url"},
.args = {"arg"},
.doc = R"(
Download the specified URL and return the path of the downloaded file.
`arg` can be either a string denoting the URL, or an attribute set with the following attributes:
- `url`
The URL of the file to download.
- `name` (default: the last path component of the URL)
A name for the file in the store. This can be useful if the URL has any
characters that are invalid for the store.
Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
)",
@ -519,11 +568,11 @@ static RegisterPrimOp primop_fetchTarball({
.doc = R"(
Download the specified URL, unpack it and return the path of the
unpacked tree. The file must be a tape archive (`.tar`) compressed
with `gzip`, `bzip2` or `xz`. The top-level path component of the
files in the tarball is removed, so it is best if the tarball
contains a single directory at top level. The typical use of the
function is to obtain external Nix expression dependencies, such as
a particular version of Nixpkgs, e.g.
with `gzip`, `bzip2` or `xz`. If the tarball consists of a
single directory, then the top-level path component of the files
in the tarball is removed. The typical use of the function is to
obtain external Nix expression dependencies, such as a
particular version of Nixpkgs, e.g.
```nix
with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
@ -598,6 +647,8 @@ static RegisterPrimOp primop_fetchGit({
[Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
This option has no effect once `shallow` cloning is enabled.
By default, the `ref` value is prefixed with `refs/heads/`.
As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`.
@ -615,23 +666,25 @@ static RegisterPrimOp primop_fetchGit({
- `shallow` (default: `false`)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
- `allRefs`
Whether to fetch all references of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(by default only `rev`s from the specified `ref` are supported).
This option has no effect once `shallow` cloning is enabled.
- `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`)
Whether to check `rev` for a signature matching `publicKey` or `publicKeys`.
If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes.
Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
- `publicKey`
The public key against which `rev` is verified if `verifyCommit` is enabled.
Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
- `keytype` (default: `"ssh-ed25519"`)
@ -643,7 +696,7 @@ static RegisterPrimOp primop_fetchGit({
- `"ssh-ed25519"`
- `"ssh-ed25519-sk"`
- `"ssh-rsa"`
Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
- `publicKeys`
@ -657,7 +710,7 @@ static RegisterPrimOp primop_fetchGit({
}
```
Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches).
Here are some examples of how to use `fetchGit`.

4
src/libexpr/primops/fromTOML.cc
View file

@ -1,10 +1,10 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "../../toml11/toml.hpp"
#include <sstream>
#include <toml.hpp>
namespace nix {
static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val)

12
src/libexpr/primops/meson.build Normal file
View file

@ -0,0 +1,12 @@
generated_headers += gen_header.process(
'derivation.nix',
preserve_path_from: meson.project_source_root(),
)
sources += files(
'context.cc',
'fetchClosure.cc',
'fetchMercurial.cc',
'fetchTree.cc',
'fromTOML.cc',
)

2
src/libexpr/print-ambiguous.cc
View file

@ -94,7 +94,7 @@ void printAmbiguous(
break;
default:
printError("Nix evaluator internal error: printAmbiguous: invalid value type");
abort();
unreachable();
}
}

67
src/libexpr/print.cc
View file

@ -1,5 +1,6 @@
#include <limits>
#include <unordered_set>
#include <sstream>
#include "print.hh"
#include "ansicolor.hh"
@ -162,8 +163,8 @@ private:
EvalState & state;
PrintOptions options;
std::optional<ValuesSeen> seen;
size_t attrsPrinted = 0;
size_t listItemsPrinted = 0;
size_t totalAttrsPrinted = 0;
size_t totalListItemsPrinted = 0;
std::string indent;
void increaseIndent()
@ -271,22 +272,36 @@ private:
void printDerivation(Value & v)
{
NixStringContext context;
std::string storePath;
if (auto i = v.attrs()->get(state.sDrvPath))
storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
std::optional<StorePath> storePath;
if (auto i = v.attrs()->get(state.sDrvPath)) {
NixStringContext context;
storePath = state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation");
}
/* This unfortunately breaks printing nested values because of
how the pretty printer is used (when pretting printing and warning
to same terminal / std stream). */
#if 0
if (storePath && !storePath->isDerivation())
warn(
"drvPath attribute '%s' is not a valid store path to a derivation, this value not work properly",
state.store->printStorePath(*storePath));
#endif
if (options.ansiColors)
output << ANSI_GREEN;
output << "«derivation";
if (!storePath.empty()) {
output << " " << storePath;
if (storePath) {
output << " " << state.store->printStorePath(*storePath);
}
output << "»";
if (options.ansiColors)
output << ANSI_NORMAL;
}
/**
* @note This may force items.
*/
bool shouldPrettyPrintAttrs(AttrVec & v)
{
if (!options.shouldPrettyPrint() || v.empty()) {
@ -303,6 +318,9 @@ private:
return true;
}
// It is ok to force the item(s) here, because they will be printed anyway.
state.forceValue(*item, item->determinePos(noPos));
// Pretty-print single-item attrsets only if they contain nested
// structures.
auto itemType = item->type();
@ -333,11 +351,13 @@ private:
auto prettyPrint = shouldPrettyPrintAttrs(sorted);
size_t currentAttrsPrinted = 0;
for (auto & i : sorted) {
printSpace(prettyPrint);
if (attrsPrinted >= options.maxAttrs) {
printElided(sorted.size() - attrsPrinted, "attribute", "attributes");
if (totalAttrsPrinted >= options.maxAttrs) {
printElided(sorted.size() - currentAttrsPrinted, "attribute", "attributes");
break;
}
@ -345,7 +365,8 @@ private:
output << " = ";
print(*i.second, depth + 1);
output << ";";
attrsPrinted++;
totalAttrsPrinted++;
currentAttrsPrinted++;
}
decreaseIndent();
@ -356,6 +377,9 @@ private:
}
}
/**
* @note This may force items.
*/
bool shouldPrettyPrintList(std::span<Value * const> list)
{
if (!options.shouldPrettyPrint() || list.empty()) {
@ -372,6 +396,9 @@ private:
return true;
}
// It is ok to force the item(s) here, because they will be printed anyway.
state.forceValue(*item, item->determinePos(noPos));
// Pretty-print single-item lists only if they contain nested
// structures.
auto itemType = item->type();
@ -390,11 +417,14 @@ private:
output << "[";
auto listItems = v.listItems();
auto prettyPrint = shouldPrettyPrintList(listItems);
size_t currentListItemsPrinted = 0;
for (auto elem : listItems) {
printSpace(prettyPrint);
if (listItemsPrinted >= options.maxListItems) {
printElided(listItems.size() - listItemsPrinted, "item", "items");
if (totalListItemsPrinted >= options.maxListItems) {
printElided(listItems.size() - currentListItemsPrinted, "item", "items");
break;
}
@ -403,7 +433,8 @@ private:
} else {
printNullptr();
}
listItemsPrinted++;
totalListItemsPrinted++;
currentListItemsPrinted++;
}
decreaseIndent();
@ -444,7 +475,7 @@ private:
else
output << "primop";
} else {
abort();
unreachable();
}
output << "»";
@ -473,7 +504,7 @@ private:
if (options.ansiColors)
output << ANSI_NORMAL;
} else {
abort();
unreachable();
}
}
@ -576,8 +607,8 @@ public:
void print(Value & v)
{
attrsPrinted = 0;
listItemsPrinted = 0;
totalAttrsPrinted = 0;
totalListItemsPrinted = 0;
indent.clear();
if (options.trackRepeated) {

26
src/libexpr/symbol-table.hh
View file

@ -7,6 +7,7 @@
#include "types.hh"
#include "chunked-vector.hh"
#include "error.hh"
namespace nix {
@ -30,9 +31,9 @@ public:
return *s == s2;
}
operator const std::string & () const
const char * c_str() const
{
return *s;
return s->c_str();
}
operator const std::string_view () const
@ -41,6 +42,11 @@ public:
}
friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
bool empty() const
{
return s->empty();
}
};
/**
@ -62,9 +68,10 @@ public:
explicit operator bool() const { return id > 0; }
bool operator<(const Symbol other) const { return id < other.id; }
auto operator<=>(const Symbol other) const { return id <=> other.id; }
bool operator==(const Symbol other) const { return id == other.id; }
bool operator!=(const Symbol other) const { return id != other.id; }
friend class std::hash<Symbol>;
};
/**
@ -109,7 +116,7 @@ public:
SymbolStr operator[](Symbol s) const
{
if (s.id == 0 || s.id > store.size())
abort();
unreachable();
return SymbolStr(store[s.id - 1]);
}
@ -128,3 +135,12 @@ public:
};
}
template<>
struct std::hash<nix::Symbol>
{
std::size_t operator()(const nix::Symbol & s) const noexcept
{
return std::hash<decltype(s.id)>{}(s.id);
}
};

4
src/libexpr/value-to-json.cc
View file

@ -22,7 +22,7 @@ json printValueAsJSON(EvalState & state, bool strict,
switch (v.type()) {
case nInt:
out = v.integer();
out = v.integer().value;
break;
case nBool:
@ -58,7 +58,7 @@ json printValueAsJSON(EvalState & state, bool strict,
out = json::object();
for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) {
try {
out[state.symbols[a->name]] = printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore);
out.emplace(state.symbols[a->name], printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore));
} catch (Error & e) {
e.addTrace(state.positions[a->pos],
HintFmt("while evaluating attribute '%1%'", state.symbols[a->name]));

2
src/libexpr/value-to-xml.cc
View file

@ -9,7 +9,7 @@
namespace nix {
static XMLAttrs singletonAttrs(const std::string & name, const std::string & value)
static XMLAttrs singletonAttrs(const std::string & name, std::string_view value)
{
XMLAttrs attrs;
attrs[name] = value;

25
src/libexpr/value.hh
View file

@ -2,13 +2,13 @@
///@file
#include <cassert>
#include <climits>
#include <span>
#include "symbol-table.hh"
#include "value/context.hh"
#include "source-path.hh"
#include "print-options.hh"
#include "checked-arithmetic.hh"
#if HAVE_BOEHMGC
#include <gc/gc_allocator.h>
@ -74,8 +74,8 @@ class EvalState;
class XMLWriter;
class Printer;
typedef int64_t NixInt;
typedef double NixFloat;
using NixInt = checked::Checked<int64_t>;
using NixFloat = double;
/**
* External values must descend from ExternalValueBase, so that
@ -112,7 +112,7 @@ class ExternalValueBase
* Compare to another value of the same type. Defaults to uncomparable,
* i.e. always false.
*/
virtual bool operator ==(const ExternalValueBase & b) const;
virtual bool operator ==(const ExternalValueBase & b) const noexcept;
/**
* Print the value as JSON. Defaults to unconvertable, i.e. throws an error
@ -286,7 +286,7 @@ public:
if (invalidIsThunk)
return nThunk;
else
abort();
unreachable();
}
inline void finishValue(InternalType newType, Payload newPayload)
@ -305,6 +305,11 @@ public:
return internalType != tUninitialized;
}
inline void mkInt(NixInt::Inner n)
{
mkInt(NixInt{n});
}
inline void mkInt(NixInt n)
{
finishValue(tInt, { .integer = n });
@ -326,9 +331,9 @@ public:
void mkStringMove(const char * s, const NixStringContext & context);
inline void mkString(const Symbol & s)
inline void mkString(const SymbolStr & s)
{
mkString(((const std::string &) s).c_str());
mkString(s.c_str());
}
void mkPath(const SourcePath & path);
@ -449,7 +454,7 @@ public:
return std::string_view(payload.string.c_str);
}
const char * const c_str() const
const char * c_str() const
{
assert(internalType == tString);
return payload.string.c_str;
@ -495,11 +500,11 @@ void Value::mkBlackhole()
#if HAVE_BOEHMGC
typedef std::vector<Value *, traceable_allocator<Value *>> ValueVector;
typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *>>> ValueMap;
typedef std::unordered_map<Symbol, Value *, std::hash<Symbol>, std::equal_to<Symbol>, traceable_allocator<std::pair<const Symbol, Value *>>> ValueMap;
typedef std::map<Symbol, ValueVector, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, ValueVector>>> ValueVectorMap;
#else
typedef std::vector<Value *> ValueVector;
typedef std::map<Symbol, Value *> ValueMap;
typedef std::unordered_map<Symbol, Value *> ValueMap;
typedef std::map<Symbol, ValueVector> ValueVectorMap;
#endif