1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-16 23:42:43 +01:00

Merge remote-tracking branch 'origin/master' into detsys-main

This commit is contained in:
Eelco Dolstra 2024-07-16 13:54:16 +02:00
commit 042c2ae3ac
222 changed files with 3295 additions and 1254 deletions

View file

@ -76,7 +76,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
if (!a) {
std::set<std::string> attrNames;
for (auto & attr : *v->attrs())
attrNames.insert(state.symbols[attr.name]);
attrNames.insert(std::string(state.symbols[attr.name]));
auto suggestions = Suggestions::bestMatches(attrNames, attr);
throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);

View file

@ -5,7 +5,6 @@
#include "symbol-table.hh"
#include <algorithm>
#include <optional>
namespace nix {
@ -28,9 +27,9 @@ struct Attr
Attr(Symbol name, Value * value, PosIdx pos = noPos)
: name(name), pos(pos), value(value) { };
Attr() { };
bool operator < (const Attr & a) const
auto operator <=> (const Attr & a) const
{
return name < a.name;
return name <=> a.name;
}
};

View file

@ -225,7 +225,7 @@ struct AttrDb
(key.first)
(symbols[key.second])
(AttrType::ListOfStrings)
(concatStringsSep("\t", l)).exec();
(dropEmptyInitThenConcatStringsSep("\t", l)).exec();
return state->db.getLastInsertedRowId();
});
@ -441,12 +441,12 @@ std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
std::string AttrCursor::getAttrPathStr() const
{
return concatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath()));
}
std::string AttrCursor::getAttrPathStr(Symbol name) const
{
return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name)));
}
Value & AttrCursor::forceValue()
@ -490,7 +490,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
auto attrNames = getAttrs();
std::set<std::string> strAttrNames;
for (auto & name : attrNames)
strAttrNames.insert(root->state.symbols[name]);
strAttrNames.insert(std::string(root->state.symbols[name]));
return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
}

View file

@ -1,7 +1,5 @@
#pragma once
#include <algorithm>
#include "error.hh"
#include "pos-idx.hh"

View file

@ -1,5 +1,4 @@
#include "users.hh"
#include "config-global.hh"
#include "globals.hh"
#include "profiles.hh"
#include "eval.hh"

View file

@ -2,6 +2,7 @@
///@file
#include "config.hh"
#include "ref.hh"
namespace nix {

View file

@ -1,6 +1,6 @@
#include "eval.hh"
#include "eval-gc.hh"
#include "eval-settings.hh"
#include "hash.hh"
#include "primops.hh"
#include "print-options.hh"
#include "exit.hh"
@ -16,7 +16,6 @@
#include "print.hh"
#include "filtering-source-accessor.hh"
#include "memory-source-accessor.hh"
#include "signals.hh"
#include "gc-small-vector.hh"
#include "url.hh"
#include "fetch-to-store.hh"
@ -24,7 +23,6 @@
#include "parser-tab.hh"
#include <algorithm>
#include <chrono>
#include <iostream>
#include <sstream>
#include <cstring>
@ -51,6 +49,8 @@
#endif
#include "strings-inline.hh"
using json = nlohmann::json;
namespace nix {
@ -217,9 +217,11 @@ static constexpr size_t BASE_ENV_SIZE = 128;
EvalState::EvalState(
const LookupPath & _lookupPath,
ref<Store> store,
const fetchers::Settings & fetchSettings,
const EvalSettings & settings,
std::shared_ptr<Store> buildStore)
: settings{settings}
: fetchSettings{fetchSettings}
, settings{settings}
, sWith(symbols.create("<with>"))
, sOutPath(symbols.create("outPath"))
, sDrvPath(symbols.create("drvPath"))
@ -557,6 +559,54 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
.doc = doc,
};
}
if (v.isLambda()) {
auto exprLambda = v.payload.lambda.fun;
std::stringstream s(std::ios_base::out);
std::string name;
auto pos = positions[exprLambda->getPos()];
std::string docStr;
if (exprLambda->name) {
name = symbols[exprLambda->name];
}
if (exprLambda->docComment) {
docStr = exprLambda->docComment.getInnerText(positions);
}
if (name.empty()) {
s << "Function ";
}
else {
s << "Function `" << name << "`";
if (pos)
s << "\\\n" ;
else
s << "\\\n";
}
if (pos) {
s << "defined at " << pos;
}
if (!docStr.empty()) {
s << "\n\n";
}
s << docStr;
s << '\0'; // for making a c string below
std::string ss = s.str();
return Doc {
.pos = pos,
.name = name,
.arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
.args = {},
.doc =
// FIXME: this leaks; make the field std::string?
strdup(ss.data()),
};
}
return {};
}
@ -633,11 +683,11 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En
if (se.isWith && !env.values[0]->isThunk()) {
// add 'with' bindings.
for (auto & j : *env.values[0]->attrs())
vm[st[j.name]] = j.value;
vm.insert_or_assign(std::string(st[j.name]), j.value);
} else {
// iterate through staticenv bindings and add them.
for (auto & i : se.vars)
vm[st[i.first]] = env.values[i.second];
vm.insert_or_assign(std::string(st[i.first]), env.values[i.second]);
}
}
}
@ -1011,7 +1061,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
if (!e)
e = parseExprFromFile(resolvedPath);
fileParseCache[resolvedPath] = e;
fileParseCache.emplace(resolvedPath, e);
try {
auto dts = debugRepl
@ -1034,8 +1084,8 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
throw;
}
fileEvalCache[resolvedPath] = v;
if (path != resolvedPath) fileEvalCache[path] = v;
fileEvalCache.emplace(resolvedPath, v);
if (path != resolvedPath) fileEvalCache.emplace(path, v);
}
@ -1338,7 +1388,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
if (!(j = vAttrs->attrs()->get(name))) {
std::set<std::string> allAttrNames;
for (auto & attr : *vAttrs->attrs())
allAttrNames.insert(state.symbols[attr.name]);
allAttrNames.insert(std::string(state.symbols[attr.name]));
auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
state.error<EvalError>("attribute '%1%' missing", state.symbols[name])
.atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow();
@ -1365,6 +1415,22 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
v = *vAttrs;
}
Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs)
{
Value vTmp;
Symbol name = getName(attrPath[attrPath.size() - 1], state, env);
if (attrPath.size() == 1) {
e->eval(state, env, vTmp);
} else {
ExprSelect init(*this);
init.attrPath.pop_back();
init.eval(state, env, vTmp);
}
attrs = vTmp;
return name;
}
void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
{
@ -1496,7 +1562,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
if (!lambda.formals->has(i.name)) {
std::set<std::string> formalNames;
for (auto & formal : lambda.formals->formals)
formalNames.insert(symbols[formal.name]);
formalNames.insert(std::string(symbols[formal.name]));
auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]);
error<TypeError>("function '%1%' called with unexpected argument '%2%'",
(lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
@ -2826,13 +2892,37 @@ Expr * EvalState::parse(
const SourcePath & basePath,
std::shared_ptr<StaticEnv> & staticEnv)
{
auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, rootFS, exprSymbols);
DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath
DocCommentMap *docComments = &tmpDocComments;
if (auto sourcePath = std::get_if<SourcePath>(&origin)) {
auto [it, _] = positionToDocComment.try_emplace(*sourcePath);
docComments = &it->second;
}
auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS, exprSymbols);
result->bindVars(*this, staticEnv);
return result;
}
DocComment EvalState::getDocCommentForPos(PosIdx pos)
{
auto pos2 = positions[pos];
auto path = pos2.getSourcePath();
if (!path)
return {};
auto table = positionToDocComment.find(*path);
if (table == positionToDocComment.end())
return {};
auto it = table->second.find(pos);
if (it == table->second.end())
return {};
return it->second;
}
std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
{
@ -2842,7 +2932,7 @@ std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx &
}
bool ExternalValueBase::operator==(const ExternalValueBase & b) const
bool ExternalValueBase::operator==(const ExternalValueBase & b) const noexcept
{
return false;
}

View file

@ -3,21 +3,21 @@
#include "attr-set.hh"
#include "eval-error.hh"
#include "eval-gc.hh"
#include "types.hh"
#include "value.hh"
#include "nixexpr.hh"
#include "symbol-table.hh"
#include "config.hh"
#include "experimental-features.hh"
#include "position.hh"
#include "pos-table.hh"
#include "source-accessor.hh"
#include "search-path.hh"
#include "repl-exit-status.hh"
#include "ref.hh"
#include <map>
#include <optional>
#include <unordered_map>
#include <mutex>
#include <functional>
namespace nix {
@ -30,6 +30,7 @@ namespace nix {
constexpr size_t maxPrimOpArity = 8;
class Store;
namespace fetchers { struct Settings; }
struct EvalSettings;
class EvalState;
class StorePath;
@ -43,7 +44,7 @@ namespace eval_cache {
/**
* Function that implements a primop.
*/
typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v);
/**
* Info about a primitive operation, and its implementation
@ -84,7 +85,7 @@ struct PrimOp
/**
* Implementation of the primop.
*/
std::function<std::remove_pointer<PrimOpFun>::type> fun;
std::function<PrimOpFun> fun;
/**
* Optional experimental for this to be gated on.
@ -129,6 +130,8 @@ struct Constant
typedef std::map<std::string, Value *> ValMap;
#endif
typedef std::map<PosIdx, DocComment> DocCommentMap;
struct Env
{
Env * up;
@ -162,6 +165,7 @@ struct DebugTrace {
class EvalState : public std::enable_shared_from_this<EvalState>
{
public:
const fetchers::Settings & fetchSettings;
const EvalSettings & settings;
SymbolTable symbols;
PosTable positions;
@ -305,15 +309,15 @@ private:
/* Cache for calls to addToStore(); maps source paths to the store
paths. */
Sync<std::map<SourcePath, StorePath>> srcToStore;
Sync<std::unordered_map<SourcePath, StorePath>> srcToStore;
/**
* A cache from path names to parse trees.
*/
#if HAVE_BOEHMGC
typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
typedef std::unordered_map<SourcePath, Expr *, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
#else
typedef std::map<SourcePath, Expr *> FileParseCache;
typedef std::unordered_map<SourcePath, Expr *> FileParseCache;
#endif
FileParseCache fileParseCache;
@ -321,12 +325,18 @@ private:
* A cache from path names to values.
*/
#if HAVE_BOEHMGC
typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
typedef std::unordered_map<SourcePath, Value, std::hash<SourcePath>, std::equal_to<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
#else
typedef std::map<SourcePath, Value> FileEvalCache;
typedef std::unordered_map<SourcePath, Value> FileEvalCache;
#endif
FileEvalCache fileEvalCache;
/**
* Associate source positions of certain AST nodes with their preceding doc comment, if they have one.
* Grouped by file.
*/
std::map<SourcePath, DocCommentMap> positionToDocComment;
LookupPath lookupPath;
std::map<std::string, std::optional<std::string>> lookupPathResolved;
@ -353,6 +363,7 @@ public:
EvalState(
const LookupPath & _lookupPath,
ref<Store> store,
const fetchers::Settings & fetchSettings,
const EvalSettings & settings,
std::shared_ptr<Store> buildStore = nullptr);
~EvalState();
@ -768,6 +779,8 @@ public:
std::string_view pathArg,
PosIdx pos);
DocComment getDocCommentForPos(PosIdx pos);
private:
/**

View file

@ -342,9 +342,9 @@ std::optional<PackageInfo> getDerivation(EvalState & state, Value & v,
}
static std::string addToPath(const std::string & s1, const std::string & s2)
static std::string addToPath(const std::string & s1, std::string_view s2)
{
return s1.empty() ? s2 : s1 + "." + s2;
return s1.empty() ? std::string(s2) : s1 + "." + s2;
}

View file

@ -0,0 +1,30 @@
#include "lexer-tab.hh"
#include "lexer-helpers.hh"
#include "parser-tab.hh"
void nix::lexer::internal::initLoc(YYLTYPE * loc)
{
loc->beginOffset = loc->endOffset = 0;
}
void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
{
loc->stash();
LexerState & lexerState = *yyget_extra(yyscanner);
if (lexerState.docCommentDistance == 1) {
// Preceding token was a doc comment.
ParserLocation doc;
doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset;
ParserLocation docEnd;
docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset;
DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)};
PosIdx locPos = lexerState.at(*loc);
lexerState.positionToDocComment.emplace(locPos, docComment);
}
lexerState.docCommentDistance++;
loc->beginOffset = loc->endOffset;
loc->endOffset += len;
}

View file

@ -0,0 +1,9 @@
#pragma once
namespace nix::lexer::internal {
void initLoc(YYLTYPE * loc);
void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
} // namespace nix::lexer

View file

@ -5,7 +5,7 @@
%option stack
%option nodefault
%option nounput noyy_top_state
%option extra-type="::nix::LexerState *"
%s DEFAULT
%x STRING
@ -14,6 +14,10 @@
%x INPATH_SLASH
%x PATH_START
%top {
#include "parser-tab.hh" // YYSTYPE
#include "parser-state.hh"
}
%{
#ifdef __clang__
@ -22,28 +26,19 @@
#include "nixexpr.hh"
#include "parser-tab.hh"
#include "lexer-helpers.hh"
namespace nix {
struct LexerState;
}
using namespace nix;
using namespace nix::lexer::internal;
namespace nix {
#define CUR_POS state->at(*yylloc)
static void initLoc(YYLTYPE * loc)
{
loc->first_line = loc->last_line = 0;
loc->first_column = loc->last_column = 0;
}
static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
{
loc->stash();
loc->first_column = loc->last_column;
loc->last_column += len;
}
// we make use of the fact that the parser receives a private copy of the input
// string and can munge around in it.
static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
@ -79,7 +74,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
#define YY_USER_INIT initLoc(yylloc)
#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng);
#define PUSH_STATE(state) yy_push_state(state, yyscanner)
#define POP_STATE() yy_pop_state(yyscanner)
@ -279,9 +274,32 @@ or { return OR_KW; }
{SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
{URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
[ \t\r\n]+ /* eat up whitespace */
\#[^\r\n]* /* single-line comments */
\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */
%{
// Doc comment rule
//
// \/\*\* /**
// [^/*] reject /**/ (empty comment) and /***
// ([^*]|\*+[^*/])*\*+\/ same as the long comment rule
// ( )* zero or more non-ending sequences
// \* end(1)
// \/ end(2)
%}
\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/ /* doc comments */ {
LexerState & lexerState = *yyget_extra(yyscanner);
lexerState.docCommentDistance = 0;
lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset;
lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset;
}
%{
// The following rules have docCommentDistance--
// This compensates for the docCommentDistance++ which happens by default to
// make all the other rules invalidate the doc comment.
%}
[ \t\r\n]+ /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; }
\#[^\r\n]* /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; }
{ANY} {
/* Don't return a negative number, as this will cause

View file

@ -139,6 +139,7 @@ sources = files(
'function-trace.cc',
'get-drvs.cc',
'json-to-value.cc',
'lexer-helpers.cc',
'nixexpr.cc',
'paths.cc',
'primops.cc',
@ -165,6 +166,7 @@ headers = [config_h] + files(
'gc-small-vector.hh',
'get-drvs.hh',
'json-to-value.hh',
# internal: 'lexer-helpers.hh',
'nixexpr.hh',
'parser-state.hh',
'pos-idx.hh',

View file

@ -1,5 +1,4 @@
#include "nixexpr.hh"
#include "derivations.hh"
#include "eval.hh"
#include "symbol-table.hh"
#include "util.hh"
@ -8,6 +7,8 @@
#include <cstdlib>
#include <sstream>
#include "strings-inline.hh"
namespace nix {
unsigned long Expr::nrExprs = 0;
@ -582,6 +583,22 @@ std::string ExprLambda::showNamePos(const EvalState & state) const
return fmt("%1% at %2%", id, state.positions[pos]);
}
void ExprLambda::setDocComment(DocComment docComment) {
// RFC 145 specifies that the innermost doc comment wins.
// See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement
if (!this->docComment) {
this->docComment = docComment;
// Curried functions are defined by putting a function directly
// in the body of another function. To render docs for those, we
// need to propagate the doc comment to the innermost function.
//
// If we have our own comment, we've already propagated it, so this
// belongs in the same conditional.
body->setDocComment(docComment);
}
};
/* Position table. */
@ -626,4 +643,22 @@ size_t SymbolTable::totalSize() const
return n;
}
std::string DocComment::getInnerText(const PosTable & positions) const {
auto beginPos = positions[begin];
auto endPos = positions[end];
auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or("");
// Strip "/**" and "*/"
constexpr size_t prefixLen = 3;
constexpr size_t suffixLen = 2;
std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen);
if (docStr.empty())
return {};
// Turn the now missing "/**" into indentation
docStr = " " + docStr;
// Strip indentation (for the whole, potentially multi-line string)
docStr = stripIndentation(docStr);
return docStr;
}
}

View file

@ -6,21 +6,58 @@
#include "value.hh"
#include "symbol-table.hh"
#include "error.hh"
#include "position.hh"
#include "eval-error.hh"
#include "pos-idx.hh"
#include "pos-table.hh"
namespace nix {
struct Env;
struct Value;
class EvalState;
class PosTable;
struct Env;
struct ExprWith;
struct StaticEnv;
struct Value;
/**
* A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md)
*
* Note that this does not implement the following:
* - argument attribute names ("formals"): TBD
* - argument names: these are internal to the function and their names may not be optimal for documentation
* - function arity (degree of currying or number of ':'s):
* - Functions returning partially applied functions have a higher arity
* than can be determined locally and without evaluation.
* We do not want to present false data.
* - Some functions should be thought of as transformations of other
* functions. For instance `overlay -> overlay -> overlay` is the simplest
* way to understand `composeExtensions`, but its implementation looks like
* `f: g: final: prev: <...>`. The parameters `final` and `prev` are part
* of the overlay concept, while distracting from the function's purpose.
*/
struct DocComment {
/**
* Start of the comment, including the opening, ie `/` and `**`.
*/
PosIdx begin;
/**
* Position right after the final asterisk and `/` that terminate the comment.
*/
PosIdx end;
/**
* Whether the comment is set.
*
* A `DocComment` is small enough that it makes sense to pass by value, and
* therefore baking optionality into it is also useful, to avoiding the memory
* overhead of `std::optional`.
*/
operator bool() const { return static_cast<bool>(begin); }
std::string getInnerText(const PosTable & positions) const;
};
/**
* An attribute path is a sequence of attribute names.
@ -57,6 +94,7 @@ struct Expr
virtual void eval(EvalState & state, Env & env, Value & v);
virtual Value * maybeThunk(EvalState & state, Env & env);
virtual void setName(Symbol name);
virtual void setDocComment(DocComment docComment) { };
virtual PosIdx getPos() const { return noPos; }
};
@ -159,6 +197,17 @@ struct ExprSelect : Expr
ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
PosIdx getPos() const override { return pos; }
/**
* Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl.
*
* @param[out] v The attribute set that should contain the last attribute name (if it exists).
* @return The last attribute name in `attrPath`
*
* @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist.
*/
Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs);
COMMON_METHODS
};
@ -281,6 +330,8 @@ struct ExprLambda : Expr
Symbol arg;
Formals * formals;
Expr * body;
DocComment docComment;
ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
: pos(pos), arg(arg), formals(formals), body(body)
{
@ -293,6 +344,7 @@ struct ExprLambda : Expr
std::string showNamePos(const EvalState & state) const;
inline bool hasFormals() const { return formals != nullptr; }
PosIdx getPos() const override { return pos; }
virtual void setDocComment(DocComment docComment) override;
COMMON_METHODS
};

View file

@ -1,6 +1,8 @@
#pragma once
///@file
#include <limits>
#include "eval.hh"
namespace nix {
@ -20,25 +22,59 @@ struct StringToken
struct ParserLocation
{
int first_line, first_column;
int last_line, last_column;
int beginOffset;
int endOffset;
// backup to recover from yyless(0)
int stashed_first_column, stashed_last_column;
int stashedBeginOffset, stashedEndOffset;
void stash() {
stashed_first_column = first_column;
stashed_last_column = last_column;
stashedBeginOffset = beginOffset;
stashedEndOffset = endOffset;
}
void unstash() {
first_column = stashed_first_column;
last_column = stashed_last_column;
beginOffset = stashedBeginOffset;
endOffset = stashedEndOffset;
}
/** Latest doc comment position, or 0. */
int doc_comment_first_column, doc_comment_last_column;
};
struct LexerState
{
/**
* Tracks the distance to the last doc comment, in terms of lexer tokens.
*
* The lexer sets this to 0 when reading a doc comment, and increments it
* for every matched rule; see `lexer-helpers.cc`.
* Whitespace and comment rules decrement the distance, so that they result
* in a net 0 change in distance.
*/
int docCommentDistance = std::numeric_limits<int>::max();
/**
* The location of the last doc comment.
*
* (stashing fields are not used)
*/
ParserLocation lastDocCommentLoc;
/**
* @brief Maps some positions to a DocComment, where the comment is relevant to the location.
*/
std::map<PosIdx, DocComment> & positionToDocComment;
PosTable & positions;
PosTable::Origin origin;
PosIdx at(const ParserLocation & loc);
};
struct ParserState
{
const LexerState & lexerState;
SymbolTable & symbols;
PosTable & positions;
Expr * result;
@ -270,9 +306,14 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
return new ExprConcatStrings(pos, true, es2);
}
inline PosIdx LexerState::at(const ParserLocation & loc)
{
return positions.add(origin, loc.beginOffset);
}
inline PosIdx ParserState::at(const ParserLocation & loc)
{
return positions.add(origin, loc.first_column);
return positions.add(origin, loc.beginOffset);
}
}

View file

@ -31,8 +31,25 @@
#define YY_DECL int yylex \
(YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
// For efficiency, we only track offsets; not line,column coordinates
# define YYLLOC_DEFAULT(Current, Rhs, N) \
do \
if (N) \
{ \
(Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset; \
(Current).endOffset = YYRHSLOC (Rhs, N).endOffset; \
} \
else \
{ \
(Current).beginOffset = (Current).endOffset = \
YYRHSLOC (Rhs, 0).endOffset; \
} \
while (0)
namespace nix {
typedef std::map<PosIdx, DocComment> DocCommentMap;
Expr * parseExprFromBuf(
char * text,
size_t length,
@ -41,6 +58,7 @@ Expr * parseExprFromBuf(
SymbolTable & symbols,
const EvalSettings & settings,
PosTable & positions,
DocCommentMap & docComments,
const ref<SourceAccessor> rootFS,
const Expr::AstSymbols & astSymbols);
@ -65,8 +83,7 @@ using namespace nix;
void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
{
if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
loc->first_column = loc->last_column;
loc->first_line = loc->last_line;
loc->beginOffset = loc->endOffset;
}
throw ParseError({
.msg = HintFmt(error),
@ -74,6 +91,14 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
});
}
#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos))
static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) {
auto it = lexerState.positionToDocComment.find(start);
if (it != lexerState.positionToDocComment.end()) {
lambda->setDocComment(it->second);
}
}
%}
@ -119,6 +144,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char *
%token IND_STRING_OPEN IND_STRING_CLOSE
%token ELLIPSIS
%right IMPL
%left OR
%left AND
@ -140,18 +166,28 @@ expr: expr_function;
expr_function
: ID ':' expr_function
{ $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); }
{ auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3);
$$ = me;
SET_DOC_POS(me, @1);
}
| '{' formals '}' ':' expr_function
{ $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); }
{ auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5);
$$ = me;
SET_DOC_POS(me, @1);
}
| '{' formals '}' '@' ID ':' expr_function
{
auto arg = state->symbols.create($5);
$$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
$$ = me;
SET_DOC_POS(me, @1);
}
| ID '@' '{' formals '}' ':' expr_function
{
auto arg = state->symbols.create($1);
$$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
$$ = me;
SET_DOC_POS(me, @1);
}
| ASSERT expr ';' expr_function
{ $$ = new ExprAssert(CUR_POS, $2, $4); }
@ -312,7 +348,22 @@ ind_string_parts
;
binds
: binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
: binds attrpath '=' expr ';' {
$$ = $1;
auto pos = state->at(@2);
auto exprPos = state->at(@4);
{
auto it = state->lexerState.positionToDocComment.find(pos);
if (it != state->lexerState.positionToDocComment.end()) {
$4->setDocComment(it->second);
state->lexerState.positionToDocComment.emplace(exprPos, it->second);
}
}
state->addAttr($$, std::move(*$2), $4, pos);
delete $2;
}
| binds INHERIT attrs ';'
{ $$ = $1;
for (auto & [i, iPos] : *$3) {
@ -431,21 +482,28 @@ Expr * parseExprFromBuf(
SymbolTable & symbols,
const EvalSettings & settings,
PosTable & positions,
DocCommentMap & docComments,
const ref<SourceAccessor> rootFS,
const Expr::AstSymbols & astSymbols)
{
yyscan_t scanner;
LexerState lexerState {
.positionToDocComment = docComments,
.positions = positions,
.origin = positions.addOrigin(origin, length),
};
ParserState state {
.lexerState = lexerState,
.symbols = symbols,
.positions = positions,
.basePath = basePath,
.origin = positions.addOrigin(origin, length),
.origin = lexerState.origin,
.rootFS = rootFS,
.s = astSymbols,
.settings = settings,
};
yylex_init(&scanner);
yylex_init_extra(&lexerState, &scanner);
Finally _destroy([&] { yylex_destroy(scanner); });
yy_scan_buffer(text, length, scanner);

View file

@ -28,20 +28,15 @@ public:
return id > 0;
}
bool operator<(const PosIdx other) const
auto operator<=>(const PosIdx other) const
{
return id < other.id;
return id <=> other.id;
}
bool operator==(const PosIdx other) const
{
return id == other.id;
}
bool operator!=(const PosIdx other) const
{
return id != other.id;
}
};
inline PosIdx noPos = {};

View file

@ -1,10 +1,8 @@
#pragma once
#include <cinttypes>
#include <numeric>
#include <cstdint>
#include <vector>
#include "chunked-vector.hh"
#include "pos-idx.hh"
#include "position.hh"
#include "sync.hh"

View file

@ -1,4 +1,3 @@
#include "archive.hh"
#include "derivations.hh"
#include "downstream-placeholder.hh"
#include "eval-inline.hh"
@ -1225,7 +1224,7 @@ static void derivationStrictInternal(
for (auto & i : attrs->lexicographicOrder(state.symbols)) {
if (i->name == state.sIgnoreNulls) continue;
const std::string & key = state.symbols[i->name];
auto key = state.symbols[i->name];
vomit("processing attribute '%1%'", key);
auto handleHashMode = [&](const std::string_view s) {
@ -1309,7 +1308,7 @@ static void derivationStrictInternal(
if (i->name == state.sStructuredAttrs) continue;
(*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context);
jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context));
if (i->name == state.sBuilder)
drv.builder = state.forceString(*i->value, context, pos, context_below);

View file

@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
attrs.insert_or_assign("name", std::string(name));
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::Input::fromAttrs(std::move(attrs));
auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
auto [storePath, input2] = input.fetchToStore(state.store);

View file

@ -85,7 +85,7 @@ static void fetchTree(
Value & v,
const FetchTreeParams & params = FetchTreeParams{}
) {
fetchers::Input input;
fetchers::Input input { state.fetchSettings };
NixStringContext context;
std::optional<std::string> type;
if (params.isFetchGit) type = "git";
@ -148,7 +148,7 @@ static void fetchTree(
"attribute 'name' isnt supported in call to 'fetchTree'"
).atPos(pos).debugThrow();
input = fetchers::Input::fromAttrs(std::move(attrs));
input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
} else {
auto url = state.coerceToString(pos, *args[0], context,
"while evaluating the first argument passed to the fetcher",
@ -161,9 +161,9 @@ static void fetchTree(
if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
attrs.emplace("exportIgnore", Explicit<bool>{true});
}
input = fetchers::Input::fromAttrs(std::move(attrs));
input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
} else {
input = fetchers::Input::fromURL(url);
input = fetchers::Input::fromURL(state.fetchSettings, url);
}
}

View file

@ -30,9 +30,9 @@ public:
return *s == s2;
}
operator const std::string & () const
const char * c_str() const
{
return *s;
return s->c_str();
}
operator const std::string_view () const
@ -41,6 +41,11 @@ public:
}
friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
bool empty() const
{
return s->empty();
}
};
/**
@ -62,9 +67,8 @@ public:
explicit operator bool() const { return id > 0; }
bool operator<(const Symbol other) const { return id < other.id; }
auto operator<=>(const Symbol other) const { return id <=> other.id; }
bool operator==(const Symbol other) const { return id == other.id; }
bool operator!=(const Symbol other) const { return id != other.id; }
};
/**

View file

@ -58,7 +58,7 @@ json printValueAsJSON(EvalState & state, bool strict,
out = json::object();
for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) {
try {
out[state.symbols[a->name]] = printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore);
out.emplace(state.symbols[a->name], printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore));
} catch (Error & e) {
e.addTrace(state.positions[a->pos],
HintFmt("while evaluating attribute '%1%'", state.symbols[a->name]));

View file

@ -9,7 +9,7 @@
namespace nix {
static XMLAttrs singletonAttrs(const std::string & name, const std::string & value)
static XMLAttrs singletonAttrs(const std::string & name, std::string_view value)
{
XMLAttrs attrs;
attrs[name] = value;

View file

@ -2,7 +2,6 @@
///@file
#include <cassert>
#include <climits>
#include <span>
#include "symbol-table.hh"
@ -112,7 +111,7 @@ class ExternalValueBase
* Compare to another value of the same type. Defaults to uncomparable,
* i.e. always false.
*/
virtual bool operator ==(const ExternalValueBase & b) const;
virtual bool operator ==(const ExternalValueBase & b) const noexcept;
/**
* Print the value as JSON. Defaults to unconvertable, i.e. throws an error