diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 9ad02b5f0..74eafddc3 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -9,7 +9,6 @@ #include "store-api.hh" #include "shared.hh" #include "flake/flake.hh" -#include "eval-cache.hh" #include "url.hh" #include "registry.hh" @@ -222,10 +221,8 @@ void completeFlakeRefWithFragment( // FIXME: do tilde expansion. auto flakeRef = parseFlakeRef(flakeRefS, absPath(".")); - auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); - - auto root = evalCache->getRoot(); + auto lockedFlake = lockFlake(*evalState, flakeRef, lockFlags); + auto rootValue = getFlakeValue(*evalState, lockedFlake); /* Complete 'fragment' relative to all the attrpath prefixes as well as the root of the @@ -243,12 +240,18 @@ void completeFlakeRefWithFragment( attrPath.pop_back(); } - auto attr = root->findAlongAttrPath(attrPath); - if (!attr) continue; + auto cachedFieldNames = rootValue->getCache().listChildrenAtPath(evalState->symbols, attrPath); - for (auto & attr2 : attr->getAttrs()) { - if (hasPrefix(attr2, lastAttr)) { - auto attrPath2 = attr->getAttrPath(attr2); + if (!cachedFieldNames) { + auto accessResult = evalState->getOptionalAttrField(*rootValue, attrPath, noPos); + if (accessResult.error) continue; + cachedFieldNames = evalState->listAttrFields(*accessResult.value, *accessResult.pos); + } + + for (auto & lastFieldName : *cachedFieldNames) { + if (hasPrefix(lastFieldName, lastAttr)) { + auto attrPath2 = attrPath; + attrPath2.push_back(lastFieldName); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2)); @@ -260,8 +263,8 @@ void completeFlakeRefWithFragment( attrpaths. */ if (fragment.empty()) { for (auto & attrPath : defaultFlakeAttrPaths) { - auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); - if (!attr) continue; + auto accessResult = evalState->getOptionalAttrField(*rootValue, {evalState->symbols.create(attrPath)}, noPos); + if (accessResult.error) continue; completions->add(flakeRefS + "#"); } } @@ -311,24 +314,6 @@ Buildable Installable::toBuildable() return std::move(buildables[0]); } -std::vector, std::string>> -Installable::getCursors(EvalState & state) -{ - auto evalCache = - std::make_shared(std::nullopt, state, - [&]() { return toValue(state).first; }); - return {{evalCache->getRoot(), ""}}; -} - -std::pair, std::string> -Installable::getCursor(EvalState & state) -{ - auto cursors = getCursors(state); - if (cursors.empty()) - throw Error("cannot find flake attribute '%s'", what()); - return cursors[0]; -} - struct InstallableStorePath : Installable { ref store; @@ -399,11 +384,11 @@ struct InstallableAttrPath : InstallableValue std::string what() override { return attrPath; } - std::pair toValue(EvalState & state) override + std::vector toValues(EvalState & state) override { auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); state.forceValue(*vRes); - return {vRes, pos}; + return {{vRes, pos, attrPath}}; } virtual std::vector toDerivations() override; @@ -411,7 +396,7 @@ struct InstallableAttrPath : InstallableValue std::vector InstallableAttrPath::toDerivations() { - auto v = toValue(*state).first; + auto v = toValue(*state).value; Bindings & autoArgs = *cmd.getAutoArgs(*state); @@ -445,45 +430,7 @@ std::vector InstallableFlake::getActualAttrPaths() Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) { - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value); - - return aOutputs->value; -} - -ref openEvalCache( - EvalState & state, - std::shared_ptr lockedFlake) -{ - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() - { - /* For testing whether the evaluation cache is - complete. */ - if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0") - throw Error("not everything is cached, but evaluation is not allowed"); - - auto vFlake = state.allocValue(); - flake::callFlake(state, *lockedFlake, *vFlake); - - state.forceAttrs(*vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - return aOutputs->value; - }); + return getFlakeValue(state, lockedFlake); } static std::string showAttrPaths(const std::vector & paths) @@ -500,29 +447,20 @@ std::tuple InstallableF { auto lockedFlake = getLockedFlake(); - auto cache = openEvalCache(*state, lockedFlake); - auto root = cache->getRoot(); + auto flakeValue = toValue(*state); - for (auto & attrPath : getActualAttrPaths()) { - auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath)); - if (!attr) continue; + auto rawDrvInfo = getDerivation(*state, *flakeValue.value, false); + if (!rawDrvInfo) + throw Error("flake output attribute '%s' is not a derivation", flakeValue.positionInfo); - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); + auto drvInfo = InstallableValue::DerivationInfo + { + state->store->parseStorePath(rawDrvInfo->queryDrvPath()), + state->store->maybeParseStorePath(rawDrvInfo->queryOutPath()), + rawDrvInfo->queryOutputName() + }; - auto drvPath = attr->forceDerivation(); - - auto drvInfo = DerivationInfo{ - std::move(drvPath), - state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()), - attr->getAttr(state->sOutputName)->getString() - }; - - return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)}; - } - - throw Error("flake '%s' does not provide attribute %s", - flakeRef, showAttrPaths(getActualAttrPaths())); + return {flakeValue.positionInfo, lockedFlake->flake.lockedRef, std::move(drvInfo)}; } std::vector InstallableFlake::toDerivations() @@ -532,8 +470,20 @@ std::vector InstallableFlake::toDerivations() return res; } -std::pair InstallableFlake::toValue(EvalState & state) +Installable::ValueInfo InstallableFlake::toValue(EvalState & state) { + auto values = toValues(state); + if (values.empty()) + throw Error("flake '%s' does not provide attribute %s", + flakeRef, showAttrPaths(getActualAttrPaths())); + return values[0]; + +} + +std::vector +InstallableFlake::toValues(EvalState & state) +{ + std::vector res; auto lockedFlake = getLockedFlake(); auto vOutputs = getFlakeOutputs(state, *lockedFlake); @@ -544,30 +494,11 @@ std::pair InstallableFlake::toValue(EvalState & state) try { auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); state.forceValue(*v); - return {v, pos}; + res.push_back({v, pos, attrPath}); } catch (AttrPathNotFound & e) { } } - throw Error("flake '%s' does not provide attribute %s", - flakeRef, showAttrPaths(getActualAttrPaths())); -} - -std::vector, std::string>> -InstallableFlake::getCursors(EvalState & state) -{ - auto evalCache = openEvalCache(state, - std::make_shared(lockFlake(state, flakeRef, lockFlags))); - - auto root = evalCache->getRoot(); - - std::vector, std::string>> res; - - for (auto & attrPath : getActualAttrPaths()) { - auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back({attr, attrPath}); - } - return res; } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index f37b3f829..e15194aac 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -54,7 +54,23 @@ struct Installable App toApp(EvalState & state); - virtual std::pair toValue(EvalState & state) + struct ValueInfo { + Value * value; + Pos pos; + string positionInfo; + }; + + virtual ValueInfo toValue(EvalState & state) + { + auto values = toValues(state); + if (values.empty()) + throw Error("Installable '%s' does not provide a default value", + what()); + return values[0]; + } + + virtual std::vector + toValues(EvalState & state) { throw Error("argument '%s' cannot be evaluated", what()); } @@ -66,11 +82,6 @@ struct Installable return {}; } - virtual std::vector, std::string>> - getCursors(EvalState & state); - - std::pair, std::string> - getCursor(EvalState & state); virtual FlakeRef nixpkgsFlakeRef() const { @@ -120,18 +131,14 @@ struct InstallableFlake : InstallableValue std::vector toDerivations() override; - std::pair toValue(EvalState & state) override; + ValueInfo toValue(EvalState & state) override; - std::vector, std::string>> - getCursors(EvalState & state) override; + std::vector + toValues(EvalState & state) override; std::shared_ptr getLockedFlake() const; FlakeRef nixpkgsFlakeRef() const override; }; -ref openEvalCache( - EvalState & state, - std::shared_ptr lockedFlake); - } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 9dd557205..f193cf3b2 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -73,11 +73,12 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr if (attr.empty()) throw Error("empty attribute name in selection path '%1%'", attrPath); - Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); - if (a == v->attrs->end()) + Value * vRes = state.allocValue(); + auto getResult = state.getOptionalAttrField(*v, {state.symbols.create(attr)}, pos, *vRes); + if (getResult.error) throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); - v = &*a->value; - pos = *a->pos; + v = vRes; + pos = *getResult.pos; } else { diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 6d68e5df3..501507f4e 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -38,6 +38,10 @@ public: private: size_t size_, capacity_; + +public: + +private: Attr attrs[0]; Bindings(size_t capacity) : size_(0), capacity_(capacity) { } diff --git a/src/libexpr/context.cc b/src/libexpr/context.cc new file mode 100644 index 000000000..0ca96f8ae --- /dev/null +++ b/src/libexpr/context.cc @@ -0,0 +1,21 @@ +#include "context.hh" + +namespace nix { + +/* Decode a context string ‘!!’ into a pair . */ +std::pair decodeContext(std::string_view s) +{ + if (s.at(0) == '!') { + size_t index = s.find("!", 1); + return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))}; + } else + return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""}; +} + +std::string encodeContext(std::string_view name, std::string_view path) +{ + return "!" + std::string(name) + "!" + std::string(path); +} + +} diff --git a/src/libexpr/context.hh b/src/libexpr/context.hh new file mode 100644 index 000000000..65f12de93 --- /dev/null +++ b/src/libexpr/context.hh @@ -0,0 +1,11 @@ +#include "util.hh" + +namespace nix { + +/* Decode a context string ‘!!’ into a pair . */ +std::pair decodeContext(std::string_view s); + +std::string encodeContext(std::string_view name, std::string_view path); + +} diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc deleted file mode 100644 index 98d91c905..000000000 --- a/src/libexpr/eval-cache.cc +++ /dev/null @@ -1,629 +0,0 @@ -#include "eval-cache.hh" -#include "sqlite.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "store-api.hh" - -namespace nix::eval_cache { - -static const char * schema = R"sql( -create table if not exists Attributes ( - parent integer not null, - name text, - type integer not null, - value text, - context text, - primary key (parent, name) -); -)sql"; - -struct AttrDb -{ - std::atomic_bool failed{false}; - - struct State - { - SQLite db; - SQLiteStmt insertAttribute; - SQLiteStmt insertAttributeWithContext; - SQLiteStmt queryAttribute; - SQLiteStmt queryAttributes; - std::unique_ptr txn; - }; - - std::unique_ptr> _state; - - AttrDb(const Hash & fingerprint) - : _state(std::make_unique>()) - { - auto state(_state->lock()); - - Path cacheDir = getCacheDir() + "/nix/eval-cache-v2"; - createDirs(cacheDir); - - Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; - - state->db = SQLite(dbPath); - state->db.isCache(); - state->db.exec(schema); - - state->insertAttribute.create(state->db, - "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); - - state->insertAttributeWithContext.create(state->db, - "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)"); - - state->queryAttribute.create(state->db, - "select rowid, type, value, context from Attributes where parent = ? and name = ?"); - - state->queryAttributes.create(state->db, - "select name from Attributes where parent = ?"); - - state->txn = std::make_unique(state->db); - } - - ~AttrDb() - { - try { - auto state(_state->lock()); - if (!failed) - state->txn->commit(); - state->txn.reset(); - } catch (...) { - ignoreException(); - } - } - - template - AttrId doSQLite(F && fun) - { - if (failed) return 0; - try { - return fun(); - } catch (SQLiteError &) { - ignoreException(); - failed = true; - return 0; - } - } - - AttrId setAttrs( - AttrKey key, - const std::vector & attrs) - { - return doSQLite([&]() - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::FullAttrs) - (0, false).exec(); - - AttrId rowId = state->db.getLastInsertedRowId(); - assert(rowId); - - for (auto & attr : attrs) - state->insertAttribute.use() - (rowId) - (attr) - (AttrType::Placeholder) - (0, false).exec(); - - return rowId; - }); - } - - AttrId setString( - AttrKey key, - std::string_view s, - const char * * context = nullptr) - { - return doSQLite([&]() - { - auto state(_state->lock()); - - if (context) { - std::string ctx; - for (const char * * p = context; *p; ++p) { - if (p != context) ctx.push_back(' '); - ctx.append(*p); - } - state->insertAttributeWithContext.use() - (key.first) - (key.second) - (AttrType::String) - (s) - (ctx).exec(); - } else { - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::String) - (s).exec(); - } - - return state->db.getLastInsertedRowId(); - }); - } - - AttrId setBool( - AttrKey key, - bool b) - { - return doSQLite([&]() - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::Bool) - (b ? 1 : 0).exec(); - - return state->db.getLastInsertedRowId(); - }); - } - - AttrId setPlaceholder(AttrKey key) - { - return doSQLite([&]() - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::Placeholder) - (0, false).exec(); - - return state->db.getLastInsertedRowId(); - }); - } - - AttrId setMissing(AttrKey key) - { - return doSQLite([&]() - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::Missing) - (0, false).exec(); - - return state->db.getLastInsertedRowId(); - }); - } - - AttrId setMisc(AttrKey key) - { - return doSQLite([&]() - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::Misc) - (0, false).exec(); - - return state->db.getLastInsertedRowId(); - }); - } - - AttrId setFailed(AttrKey key) - { - return doSQLite([&]() - { - auto state(_state->lock()); - - state->insertAttribute.use() - (key.first) - (key.second) - (AttrType::Failed) - (0, false).exec(); - - return state->db.getLastInsertedRowId(); - }); - } - - std::optional> getAttr( - AttrKey key, - SymbolTable & symbols) - { - auto state(_state->lock()); - - auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); - if (!queryAttribute.next()) return {}; - - auto rowId = (AttrType) queryAttribute.getInt(0); - auto type = (AttrType) queryAttribute.getInt(1); - - switch (type) { - case AttrType::Placeholder: - return {{rowId, placeholder_t()}}; - case AttrType::FullAttrs: { - // FIXME: expensive, should separate this out. - std::vector attrs; - auto queryAttributes(state->queryAttributes.use()(rowId)); - while (queryAttributes.next()) - attrs.push_back(symbols.create(queryAttributes.getStr(0))); - return {{rowId, attrs}}; - } - case AttrType::String: { - std::vector> context; - if (!queryAttribute.isNull(3)) - for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) - context.push_back(decodeContext(s)); - return {{rowId, string_t{queryAttribute.getStr(2), context}}}; - } - case AttrType::Bool: - return {{rowId, queryAttribute.getInt(2) != 0}}; - case AttrType::Missing: - return {{rowId, missing_t()}}; - case AttrType::Misc: - return {{rowId, misc_t()}}; - case AttrType::Failed: - return {{rowId, failed_t()}}; - default: - throw Error("unexpected type in evaluation cache"); - } - } -}; - -static std::shared_ptr makeAttrDb(const Hash & fingerprint) -{ - try { - return std::make_shared(fingerprint); - } catch (SQLiteError &) { - ignoreException(); - return nullptr; - } -} - -EvalCache::EvalCache( - std::optional> useCache, - EvalState & state, - RootLoader rootLoader) - : db(useCache ? makeAttrDb(*useCache) : nullptr) - , state(state) - , rootLoader(rootLoader) -{ -} - -Value * EvalCache::getRootValue() -{ - if (!value) { - debug("getting root value"); - value = allocRootValue(rootLoader()); - } - return *value; -} - -std::shared_ptr EvalCache::getRoot() -{ - return std::make_shared(ref(shared_from_this()), std::nullopt); -} - -AttrCursor::AttrCursor( - ref root, - Parent parent, - Value * value, - std::optional> && cachedValue) - : root(root), parent(parent), cachedValue(std::move(cachedValue)) -{ - if (value) - _value = allocRootValue(value); -} - -AttrKey AttrCursor::getKey() -{ - if (!parent) - return {0, root->state.sEpsilon}; - if (!parent->first->cachedValue) { - parent->first->cachedValue = root->db->getAttr( - parent->first->getKey(), root->state.symbols); - assert(parent->first->cachedValue); - } - return {parent->first->cachedValue->first, parent->second}; -} - -Value & AttrCursor::getValue() -{ - if (!_value) { - if (parent) { - auto & vParent = parent->first->getValue(); - root->state.forceAttrs(vParent); - auto attr = vParent.attrs->get(parent->second); - if (!attr) - throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); - _value = allocRootValue(attr->value); - } else - _value = allocRootValue(root->getRootValue()); - } - return **_value; -} - -std::vector AttrCursor::getAttrPath() const -{ - if (parent) { - auto attrPath = parent->first->getAttrPath(); - attrPath.push_back(parent->second); - return attrPath; - } else - return {}; -} - -std::vector AttrCursor::getAttrPath(Symbol name) const -{ - auto attrPath = getAttrPath(); - attrPath.push_back(name); - return attrPath; -} - -std::string AttrCursor::getAttrPathStr() const -{ - return concatStringsSep(".", getAttrPath()); -} - -std::string AttrCursor::getAttrPathStr(Symbol name) const -{ - return concatStringsSep(".", getAttrPath(name)); -} - -Value & AttrCursor::forceValue() -{ - debug("evaluating uncached attribute %s", getAttrPathStr()); - - auto & v = getValue(); - - try { - root->state.forceValue(v); - } catch (EvalError &) { - debug("setting '%s' to failed", getAttrPathStr()); - if (root->db) - cachedValue = {root->db->setFailed(getKey()), failed_t()}; - throw; - } - - if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { - if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), - string_t{v.string.s, {}}}; - else if (v.type() == nPath) - cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; - else if (v.type() == nBool) - cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; - else if (v.type() == nAttrs) - ; // FIXME: do something? - else - cachedValue = {root->db->setMisc(getKey()), misc_t()}; - } - - return v; -} - -std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) -{ - if (root->db) { - if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); - - if (cachedValue) { - if (auto attrs = std::get_if>(&cachedValue->second)) { - for (auto & attr : *attrs) - if (attr == name) - return std::make_shared(root, std::make_pair(shared_from_this(), name)); - return nullptr; - } else if (std::get_if(&cachedValue->second)) { - auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols); - if (attr) { - if (std::get_if(&attr->second)) - return nullptr; - else if (std::get_if(&attr->second)) { - if (forceErrors) - debug("reevaluating failed cached attribute '%s'"); - else - throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); - } else - return std::make_shared(root, - std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); - } - // Incomplete attrset, so need to fall thru and - // evaluate to see whether 'name' exists - } else - return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - } - } - - auto & v = forceValue(); - - if (v.type() != nAttrs) - return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - - auto attr = v.attrs->get(name); - - if (!attr) { - if (root->db) { - if (!cachedValue) - cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; - root->db->setMissing({cachedValue->first, name}); - } - return nullptr; - } - - std::optional> cachedValue2; - if (root->db) { - if (!cachedValue) - cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; - cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()}; - } - - return std::make_shared( - root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); -} - -std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name) -{ - return maybeGetAttr(root->state.symbols.create(name)); -} - -std::shared_ptr AttrCursor::getAttr(Symbol name, bool forceErrors) -{ - auto p = maybeGetAttr(name, forceErrors); - if (!p) - throw Error("attribute '%s' does not exist", getAttrPathStr(name)); - return p; -} - -std::shared_ptr AttrCursor::getAttr(std::string_view name) -{ - return getAttr(root->state.symbols.create(name)); -} - -std::shared_ptr AttrCursor::findAlongAttrPath(const std::vector & attrPath) -{ - auto res = shared_from_this(); - for (auto & attr : attrPath) { - res = res->maybeGetAttr(attr); - if (!res) return {}; - } - return res; -} - -std::string AttrCursor::getString() -{ - if (root->db) { - if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); - if (cachedValue && !std::get_if(&cachedValue->second)) { - if (auto s = std::get_if(&cachedValue->second)) { - debug("using cached string attribute '%s'", getAttrPathStr()); - return s->first; - } else - throw TypeError("'%s' is not a string", getAttrPathStr()); - } - } - - auto & v = forceValue(); - - if (v.type() != nString && v.type() != nPath) - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); - - return v.type() == nString ? v.string.s : v.path; -} - -string_t AttrCursor::getStringWithContext() -{ - if (root->db) { - if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); - if (cachedValue && !std::get_if(&cachedValue->second)) { - if (auto s = std::get_if(&cachedValue->second)) { - bool valid = true; - for (auto & c : s->second) { - if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) { - valid = false; - break; - } - } - if (valid) { - debug("using cached string attribute '%s'", getAttrPathStr()); - return *s; - } - } else - throw TypeError("'%s' is not a string", getAttrPathStr()); - } - } - - auto & v = forceValue(); - - if (v.type() == nString) - return {v.string.s, v.getContext()}; - else if (v.type() == nPath) - return {v.path, {}}; - else - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); -} - -bool AttrCursor::getBool() -{ - if (root->db) { - if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); - if (cachedValue && !std::get_if(&cachedValue->second)) { - if (auto b = std::get_if(&cachedValue->second)) { - debug("using cached Boolean attribute '%s'", getAttrPathStr()); - return *b; - } else - throw TypeError("'%s' is not a Boolean", getAttrPathStr()); - } - } - - auto & v = forceValue(); - - if (v.type() != nBool) - throw TypeError("'%s' is not a Boolean", getAttrPathStr()); - - return v.boolean; -} - -std::vector AttrCursor::getAttrs() -{ - if (root->db) { - if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); - if (cachedValue && !std::get_if(&cachedValue->second)) { - if (auto attrs = std::get_if>(&cachedValue->second)) { - debug("using cached attrset attribute '%s'", getAttrPathStr()); - return *attrs; - } else - throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - } - } - - auto & v = forceValue(); - - if (v.type() != nAttrs) - throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - - std::vector attrs; - for (auto & attr : *getValue().attrs) - attrs.push_back(attr.name); - std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { - return (const string &) a < (const string &) b; - }); - - if (root->db) - cachedValue = {root->db->setAttrs(getKey(), attrs), attrs}; - - return attrs; -} - -bool AttrCursor::isDerivation() -{ - auto aType = maybeGetAttr("type"); - return aType && aType->getString() == "derivation"; -} - -StorePath AttrCursor::forceDerivation() -{ - auto aDrvPath = getAttr(root->state.sDrvPath, true); - auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); - if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { - /* The eval cache contains 'drvPath', but the actual path has - been garbage-collected. So force it to be regenerated. */ - aDrvPath->forceValue(); - if (!root->state.store->isValidPath(drvPath)) - throw Error("don't know how to recreate store derivation '%s'!", - root->state.store->printStorePath(drvPath)); - } - return drvPath; -} - -} diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh deleted file mode 100644 index e23e45c94..000000000 --- a/src/libexpr/eval-cache.hh +++ /dev/null @@ -1,123 +0,0 @@ -#pragma once - -#include "sync.hh" -#include "hash.hh" -#include "eval.hh" - -#include -#include - -namespace nix::eval_cache { - -MakeError(CachedEvalError, EvalError); - -struct AttrDb; -class AttrCursor; - -class EvalCache : public std::enable_shared_from_this -{ - friend class AttrCursor; - - std::shared_ptr db; - EvalState & state; - typedef std::function RootLoader; - RootLoader rootLoader; - RootValue value; - - Value * getRootValue(); - -public: - - EvalCache( - std::optional> useCache, - EvalState & state, - RootLoader rootLoader); - - std::shared_ptr getRoot(); -}; - -enum AttrType { - Placeholder = 0, - FullAttrs = 1, - String = 2, - Missing = 3, - Misc = 4, - Failed = 5, - Bool = 6, -}; - -struct placeholder_t {}; -struct missing_t {}; -struct misc_t {}; -struct failed_t {}; -typedef uint64_t AttrId; -typedef std::pair AttrKey; -typedef std::pair>> string_t; - -typedef std::variant< - std::vector, - string_t, - placeholder_t, - missing_t, - misc_t, - failed_t, - bool - > AttrValue; - -class AttrCursor : public std::enable_shared_from_this -{ - friend class EvalCache; - - ref root; - typedef std::optional, Symbol>> Parent; - Parent parent; - RootValue _value; - std::optional> cachedValue; - - AttrKey getKey(); - - Value & getValue(); - -public: - - AttrCursor( - ref root, - Parent parent, - Value * value = nullptr, - std::optional> && cachedValue = {}); - - std::vector getAttrPath() const; - - std::vector getAttrPath(Symbol name) const; - - std::string getAttrPathStr() const; - - std::string getAttrPathStr(Symbol name) const; - - std::shared_ptr maybeGetAttr(Symbol name, bool forceErrors = false); - - std::shared_ptr maybeGetAttr(std::string_view name); - - std::shared_ptr getAttr(Symbol name, bool forceErrors = false); - - std::shared_ptr getAttr(std::string_view name); - - std::shared_ptr findAlongAttrPath(const std::vector & attrPath); - - std::string getString(); - - string_t getStringWithContext(); - - bool getBool(); - - std::vector getAttrs(); - - bool isDerivation(); - - Value & forceValue(); - - /* Force creation of the .drv file in the Nix store. */ - StorePath forceDerivation(); -}; - -} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7271776eb..534bdc15a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -433,6 +433,9 @@ EvalState::EvalState(const Strings & _searchPath, ref store) EvalState::~EvalState() { + for (auto [_, cache] : evalCache) { + cache->commit(); + } } @@ -1118,48 +1121,352 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa unsigned long nrLookups = 0; +tree_cache::AttrValue cachedValueFor(const Value& v) +{ + tree_cache::AttrValue valueToCache; + switch (v.type()) { + case nThunk: + valueToCache = tree_cache::thunk_t{}; + break; + case nNull: + case nList: + case nFunction: + case nExternal: + valueToCache = tree_cache::unknown_t{}; + break; + case nBool: + valueToCache = v.boolean; + break; + case nString: + valueToCache = tree_cache::string_t{ + v.string.s, + v.getContext() + }; + break; + case nPath: + valueToCache = tree_cache::string_t{ + v.path, + {} + }; + break; + case nAttrs: + valueToCache = tree_cache::attributeSet_t{}; + break; + case nInt: + valueToCache = v.integer; + break; + case nFloat: + valueToCache = v.fpoint; + break; + }; + return valueToCache; +} + +std::optional> ValueCache::listChildren(SymbolTable& symbols) +{ + auto ret = std::vector(); + if (rawCache) { + auto cachedValue = rawCache->getCachedValue(); + if (std::get_if(&cachedValue)) { + for (auto & fieldStr : rawCache->getChildren()) + ret.push_back(symbols.create(fieldStr)); + } + return ret; + } + return std::nullopt; +} + +std::optional> ValueCache::listChildrenAtPath(SymbolTable & symbols, const std::vector & attrPath) +{ + auto ret = std::vector(); + if (rawCache) { + auto rawChildren = rawCache->getChildrenAtPath(attrPath); + if (!rawChildren) return std::nullopt; + for (auto & fieldStr : rawChildren.value()) + ret.push_back(symbols.create(fieldStr)); + return ret; + } + return std::nullopt; +} + +std::vector EvalState::listAttrFields(Value & attrs, const Pos & pos) +{ + // First try to get it from the cache + if (auto cachedRes = attrs.getCache().listChildren(symbols)) + return cachedRes.value(); + + auto ret = std::vector(); + forceAttrs(attrs, pos); + ret.reserve(attrs.attrs->size()); + for (auto & attr : *attrs.attrs) { + ret.push_back(attr.name); + } + return ret; +} + +EvalState::AttrAccesResult EvalState::getOptionalAttrField(Value & attrs, const std::vector & selector, const Pos & pos) +{ + Value* resValue = allocValue(); + auto accessResult = getOptionalAttrField(attrs, selector, pos, *resValue); + accessResult.value = resValue; + return accessResult; +} + +ValueCache::CacheResult ValueCache::getValue(Store & store, const std::vector & selector, Value & dest) +{ + if (!rawCache) + return { NoCacheKey }; + auto resultingCursor = rawCache->findAlongAttrPath(selector); + if (!resultingCursor) + return { CacheMiss }; + + dest.setCache(ValueCache(resultingCursor)); + + auto cachedValue = resultingCursor->getCachedValue(); + return std::visit( + overloaded{ + [&](tree_cache::attributeSet_t) { return ValueCache::CacheResult{ Forward }; }, + [&](tree_cache::unknown_t) { return ValueCache::CacheResult{ UnCacheable }; }, + [&](tree_cache::thunk_t) { return ValueCache::CacheResult{ CacheMiss }; }, + [&](tree_cache::failed_t x) -> ValueCache::CacheResult {throw EvalError(x.error); }, + [&](tree_cache::missing_t x) { + return ValueCache::CacheResult{ + .returnCode = CacheHit, + .lastQueriedSymbolIfMissing = x.attrName + }; + }, + [&](tree_cache::string_t s) { + PathSet context; + for (auto& [pathName, outputName] : s.second) { + // If the cached value depends on some non-existent + // path, we need to discard it and force the evaluation + // to bring back the context in the store + if (!store.isValidPath( + store.parseStorePath(pathName))) + return ValueCache::CacheResult{UnCacheable}; + context.insert("!" + outputName + "!" + pathName); + } + mkString(dest, s.first, context); + return ValueCache::CacheResult{CacheHit}; + }, + [&](bool b) { + dest.mkBool(b); + return ValueCache::CacheResult{CacheHit}; + }, + }, + cachedValue); +} + +struct ExprCastedVar : Expr +{ + Value * v; + ExprCastedVar(Value * v) : v(v) {}; + void show(std::ostream & str) const override { + std::set active; + printValue(str, active, *v); + } + + void bindVars(const StaticEnv & env) override {} + void eval(EvalState & state, Env & env, Value & v) override { + v = std::move(*this->v); + } + Value * maybeThunk(EvalState & state, Env & env) override { + return v; + } +}; + +EvalState::AttrAccesResult EvalState::lazyGetOptionalAttrField(Value & attrs, const std::vector & selector, const Pos & pos) +{ + Value * dest = allocValue(); + auto evalCache = attrs.getCache(); + auto cacheResult = evalCache.getValue(*store, selector, *dest); + + if (cacheResult.returnCode == ValueCache::CacheHit) { + if (cacheResult.lastQueriedSymbolIfMissing) + return { + .error = + AttrAccessError{*cacheResult.lastQueriedSymbolIfMissing}, + .pos = &pos, + }; + else + return {.pos = &pos, .value = dest}; + } + + auto & newEnv(allocEnv(0)); + + auto recordAsVar = new ExprCastedVar(&attrs); + + auto accessExpr = new ExprSelect(pos, recordAsVar, selector); + dest->mkThunk (&newEnv, accessExpr); + + return { + .pos = &pos, + .value = dest, + }; + +} + +EvalState::AttrAccesResult EvalState::getOptionalAttrField(Value & attrs, const std::vector & selector, const Pos & pos, Value & dest) +{ + auto evalCache = attrs.getCache(); + + if (auto maybeRawVal = evalCache.getRawValue(); + maybeRawVal.has_value() && + !std::holds_alternative(maybeRawVal.value()) && + !std::holds_alternative(maybeRawVal.value()) + ) + return { + .error = AttrAccessError{ + .attrName = selector[0], + .illTypedValue = &attrs, + }, + .pos = &pos, + }; + + auto cacheResult = evalCache.getValue(*store, selector, dest); + + if (cacheResult.returnCode == ValueCache::CacheHit) { + if (cacheResult.lastQueriedSymbolIfMissing) + return { + .error = + AttrAccessError{*cacheResult.lastQueriedSymbolIfMissing}, + .pos = &pos, + }; + else + return {.pos = &pos, .value = &dest}; + } + + const Pos * pos2 = &pos; + Value * currentValue = &attrs; + forceValue(*currentValue, *pos2); + + auto resultingCursor = evalCache; + for (auto & name : selector) { + nrLookups++; + Bindings::iterator j; + if (currentValue->type() != nAttrs) + return { + .error = AttrAccessError{ + .attrName = name, + .illTypedValue = currentValue, + }, + .pos = pos2, + }; + if ((j = currentValue->attrs->find(name)) == currentValue->attrs->end()) + return {.error = AttrAccessError { name }, .pos = pos2 }; + currentValue = j->value; + pos2 = j->pos; + try { + forceValue(*currentValue, *pos2); + } catch (EvalError & e) { + resultingCursor.addFailedChild(name, e); + throw; + }; + resultingCursor = resultingCursor.addChild(name, *currentValue); + if (cacheResult.returnCode == ValueCache::CacheMiss && currentValue->type() == nAttrs) { + resultingCursor.addAttrSetChilds(*currentValue->attrs); + } + currentValue->setCache(resultingCursor); + if (countCalls && pos2) attrSelects[*pos2]++; + } + + dest = *currentValue; + return { .pos = pos2 }; +} + +const ValueCache ValueCache::empty = ValueCache(nullptr); + +std::optional ValueCache::getRawValue() +{ + if (!rawCache) + return std::nullopt; + return rawCache->getCachedValue(); +} + +ValueCache ValueCache::addChild(const Symbol& name, const Value& value) +{ + if (!rawCache) + return ValueCache::empty; + return ValueCache(rawCache->addChild(name, cachedValueFor(value))); +} + +ValueCache ValueCache::addFailedChild(const Symbol& name, const Error & error) +{ + if (!rawCache) return ValueCache::empty; + return ValueCache(rawCache->addChild(name, tree_cache::failed_t{ .error = error.msg() })); +} + +ValueCache ValueCache::addNumChild(SymbolTable & symbols, int idx, const Value & value) +{ + return addChild(symbols.create(std::to_string(idx)), value); +} + +void ValueCache::addAttrSetChilds(Bindings & children) +{ + if (!rawCache) return; + for (auto & attr : children) { + addChild(attr.name, *attr.value); + } + +} + +void ValueCache::addListChilds(SymbolTable & symbols, Value** elems, int listSize) +{ + if (!rawCache) return; + for (auto i = 0; i < listSize; i++) { + addNumChild(symbols, i, *elems[i]); + } +} + +void EvalState::getAttrField(Value & attrs, const std::vector & selector, const Pos & pos, Value & dest) +{ + auto missingFieldInfo = getOptionalAttrField(attrs, selector, pos, dest); + if (missingFieldInfo.error) { + if (missingFieldInfo.error->illTypedValue) { + throwTypeError("value is %1% while a set was expected", **missingFieldInfo.error->illTypedValue); + } else { + throwEvalError( + *missingFieldInfo.pos, + "attribute '%1%' missing", missingFieldInfo.error->attrName + ); + } + } +} + +Value* EvalState::getAttrField(Value & attrs, const std::vector & selector, const Pos & pos) +{ + Value* res = allocValue(); + getAttrField(attrs, selector, pos, *res); + return res; +} + void ExprSelect::eval(EvalState & state, Env & env, Value & v) { Value vTmp; - Pos * pos2 = 0; - Value * vAttrs = &vTmp; e->eval(state, env, vTmp); - try { - - for (auto & i : attrPath) { - nrLookups++; - Bindings::iterator j; - Symbol name = getName(i, state, env); - if (def) { - state.forceValue(*vAttrs, pos); - if (vAttrs->type() != nAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - { - def->eval(state, env, v); - return; - } - } else { - state.forceAttrs(*vAttrs, pos); - if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - throwEvalError(pos, "attribute '%1%' missing", name); - } - vAttrs = j->value; - pos2 = j->pos; - if (state.countCalls && pos2) state.attrSelects[*pos2]++; - } - - state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) ); - - } catch (Error & e) { - if (pos2 && pos2->file != state.sDerivationNix) - addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", - showAttrPath(state, env, attrPath)); - throw; + std::vector evaluatedAttrs; + for (auto & i : attrPath) { + evaluatedAttrs.push_back(getName(i, state, env)); } - v = *vAttrs; + try { + if (def) { + auto missingFieldInfo = state.getOptionalAttrField(vTmp, evaluatedAttrs, pos, v); + if (missingFieldInfo.error) { + def->eval(state, env, v); + return; + } + return; + } + state.getAttrField(vTmp, evaluatedAttrs, pos, v); + } catch (Error & e) { + if (pos.file != state.sDerivationNix) + addErrorTrace(e, pos, "while evaluating the attribute '%1%'", + showAttrPath(state, env, attrPath)); + throw; + } } @@ -1690,18 +1997,6 @@ string EvalState::forceString(Value & v, const Pos & pos) } -/* Decode a context string ‘!!’ into a pair . */ -std::pair decodeContext(std::string_view s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))}; - } else - return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""}; -} - - void copyContext(const Value & v, PathSet & context) { if (v.string.context) @@ -1710,7 +2005,7 @@ void copyContext(const Value & v, PathSet & context) } -std::vector> Value::getContext() +std::vector> Value::getContext() const { std::vector> res; assert(internalType == tString); @@ -1720,6 +2015,15 @@ std::vector> Value::getContext() return res; } +void Value::setCache(ValueCache cache) +{ + evalCache = cache; +} + +ValueCache Value::getCache() const +{ + return evalCache; +} string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) { @@ -1746,12 +2050,12 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) bool EvalState::isDerivation(Value & v) { - if (v.type() != nAttrs) return false; - Bindings::iterator i = v.attrs->find(sType); - if (i == v.attrs->end()) return false; - forceValue(*i->value); - if (i->value->type() != nString) return false; - return strcmp(i->value->string.s, "derivation") == 0; + auto typeAccesRes = getOptionalAttrField(v, {sType}, noPos); + if (typeAccesRes.error) + return false; + forceValue(*typeAccesRes.value); + if (typeAccesRes.value->type() != nString) return false; + return strcmp(typeAccesRes.value->string.s, "derivation") == 0; } @@ -1790,9 +2094,9 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, if (maybeString) { return *maybeString; } - auto i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string"); - return coerceToString(pos, *i->value, context, coerceMore, copyToStore); + auto accessResult = getOptionalAttrField(v, {sOutPath}, pos); + if (accessResult.error) throwTypeError(pos, "cannot coerce a set to a string"); + return coerceToString(pos, *accessResult.value, context, coerceMore, copyToStore); } if (v.type() == nExternal) @@ -2103,5 +2407,19 @@ EvalSettings evalSettings; static GlobalConfig::Register rEvalSettings(&evalSettings); +std::shared_ptr EvalState::openTreeCache(Hash cacheKey) +{ + if (auto iter = evalCache.find(cacheKey); iter != evalCache.end()) + return iter->second; + + if (!(evalSettings.useEvalCache && evalSettings.pureEval)) + return nullptr; + auto thisCache = tree_cache::Cache::tryCreate( + cacheKey, + symbols + ); + evalCache.insert({cacheKey, thisCache}); + return thisCache; +} } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e3eaed6d3..4af0ec98d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,10 +1,14 @@ #pragma once #include "attr-set.hh" +#include "context.hh" #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" #include "config.hh" +#include "hash.hh" +#include "sqlite.hh" +#include "tree-cache.hh" #include #include @@ -14,6 +18,9 @@ namespace nix { +namespace eval_cache { + class EvalCache; +} class Store; class EvalState; @@ -116,6 +123,8 @@ private: #endif FileEvalCache fileEvalCache; + std::map> evalCache; + SearchPath searchPath; std::map> searchPathResolved; @@ -131,6 +140,9 @@ public: EvalState(const Strings & _searchPath, ref store); ~EvalState(); + std::shared_ptr openCache(Hash, std::function rootLoader); + std::shared_ptr openTreeCache(Hash); + void addToSearchPath(const string & s); SearchPath getSearchPath() { return searchPath; } @@ -303,6 +315,26 @@ public: void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, Pos * pos); + struct AttrAccessError { + const Symbol attrName; + + // To distinguish between a missing field in an attribute set + // and an access to something that's not an attribute set + std::optional illTypedValue; + }; + struct AttrAccesResult { + std::optional error; + const Pos * pos; + Value* value; + }; + AttrAccesResult getOptionalAttrField(Value & attrs, const std::vector & selector, const Pos & pos, Value & dest); + AttrAccesResult getOptionalAttrField(Value & attrs, const std::vector & selector, const Pos & pos); + AttrAccesResult lazyGetOptionalAttrField(Value & attrs, const std::vector & selector, const Pos & pos); + void getAttrField(Value & attrs, const std::vector & selector, const Pos & pos, Value & dest); + Value* getAttrField(Value & attrs, const std::vector & selector, const Pos & pos); + + std::vector listAttrFields(Value & attrs, const Pos & pos); + void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos); /* Print statistics. */ diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2e94490d4..29c7a0de4 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -617,9 +617,13 @@ void callFlake(EvalState & state, , "/"), **vCallFlake); } - state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos); - state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); - state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); + vTmp1->mkApp(*vCallFlake, vLocks); + vTmp2->mkApp(vTmp1, vRootSrc); + vRes.mkApp(vTmp2, vRootSubdir); + auto fingerprint = lockedFlake.getFingerprint(); + auto evalCache = state.openTreeCache(fingerprint); + auto cacheRoot = evalCache ? evalCache->getRoot() : nullptr; + vRes.setCache(ValueCache(cacheRoot)); } static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -658,4 +662,19 @@ Fingerprint LockedFlake::getFingerprint() const Flake::~Flake() { } +Value* getFlakeValue(EvalState & state, const flake::LockedFlake lockedFlake) +{ + /* For testing whether the evaluation cache is + complete. */ + if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0") + throw Error("not everything is cached, but evaluation is not allowed"); + + auto vFlake = state.allocValue(); + flake::callFlake(state, lockedFlake, *vFlake); + + auto aOutputs = state.getAttrField(*vFlake, {state.symbols.create("outputs")}, noPos); + + return aOutputs; +} + } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 65ed1ad0a..259175e24 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -6,7 +6,6 @@ #include "value.hh" namespace nix { - class EvalState; namespace fetchers { struct Tree; } @@ -139,4 +138,5 @@ void emitTreeAttrs( const fetchers::Input & input, Value & v, bool emptyRevFallback = false); +Value* getFlakeValue(EvalState & state, const flake::LockedFlake lockedFlake); } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1a3990ea1..c8cbf8ca7 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -10,14 +10,14 @@ namespace nix { -DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) +DrvInfo::DrvInfo(EvalState & state, const string & attrPath, std::optional attrs) : state(&state), attrs(attrs), attrPath(attrPath) { } DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) - : state(&state), attrs(nullptr), attrPath("") + : state(&state), attrs(std::nullopt), attrPath("") { auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs); @@ -46,12 +46,28 @@ DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPat } +std::optional DrvInfo::queryOptionalAttr(const Symbol attrName) const +{ + // Erk + Value * mutableAttrs = const_cast(&attrs.value()); + auto accessResult = state->getOptionalAttrField(*mutableAttrs, {attrName}, noPos); + return accessResult.error ? std::nullopt : std::optional{*accessResult.value}; +} + +Value DrvInfo::queryAttr(const Symbol attrName) const +{ + Value res; + // Erk + Value * mutableAttrs = const_cast(&attrs.value()); + state->getAttrField(*mutableAttrs, {attrName}, noPos, res); + return res; +} + string DrvInfo::queryName() const { if (name == "" && attrs) { - auto i = attrs->find(state->sName); - if (i == attrs->end()) throw TypeError("derivation name missing"); - name = state->forceStringNoCtx(*i->value); + Value nameVal = queryAttr(state->sName); + name = state->forceStringNoCtx(nameVal); } return name; } @@ -60,8 +76,8 @@ string DrvInfo::queryName() const string DrvInfo::querySystem() const { if (system == "" && attrs) { - auto i = attrs->find(state->sSystem); - system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos); + auto maybeSystemVal = queryOptionalAttr(state->sSystem); + system = maybeSystemVal ? state->forceStringNoCtx(*maybeSystemVal) : "unknown"; } return system; } @@ -70,9 +86,10 @@ string DrvInfo::querySystem() const string DrvInfo::queryDrvPath() const { if (drvPath == "" && attrs) { - Bindings::iterator i = attrs->find(state->sDrvPath); + auto drvPathVal = queryOptionalAttr(state->sDrvPath); PathSet context; - drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; + if (drvPathVal) + drvPath = state->coerceToPath(noPos, *drvPathVal, context); } return drvPath; } @@ -81,10 +98,10 @@ string DrvInfo::queryDrvPath() const string DrvInfo::queryOutPath() const { if (!outPath && attrs) { - Bindings::iterator i = attrs->find(state->sOutPath); + auto val = queryOptionalAttr(state->sOutPath); PathSet context; - if (i != attrs->end()) - outPath = state->coerceToPath(*i->pos, *i->value, context); + if (val) + outPath = state->coerceToPath(noPos, *val, context); } if (!outPath) throw UnimplementedError("CA derivations are not yet supported"); @@ -96,23 +113,23 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) { if (outputs.empty()) { /* Get the ‘outputs’ list. */ - Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { - state->forceList(*i->value, *i->pos); + std::optional outputsList; + if (attrs && (outputsList = queryOptionalAttr(state->sOutputs))) { + state->forceList(*outputsList); /* For each output... */ - for (unsigned int j = 0; j < i->value->listSize(); ++j) { + for (unsigned int j = 0; j < outputsList->listSize(); ++j) { /* Evaluate the corresponding set. */ - string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); - Bindings::iterator out = attrs->find(state->symbols.create(name)); - if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value); + string name = state->forceStringNoCtx(*outputsList->listElems()[j]); + auto out = queryOptionalAttr(state->symbols.create(name)); + if (!out) continue; + state->forceAttrs(*out); /* And evaluate its ‘outPath’ attribute. */ - Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); - if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? + Value outPath; + state->getAttrField(*out, {state->sOutPath}, noPos, outPath); PathSet context; - outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context); + outputs[name] = state->coerceToPath(noPos, outPath, context); } } else outputs["out"] = queryOutPath(); @@ -140,8 +157,8 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) string DrvInfo::queryOutputName() const { if (outputName == "" && attrs) { - Bindings::iterator i = attrs->find(state->sOutputName); - outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; + auto val = queryOptionalAttr(state->sOutputName); + outputName = val ? state->forceStringNoCtx(*val) : ""; } return outputName; } @@ -151,10 +168,10 @@ Bindings * DrvInfo::getMeta() { if (meta) return meta; if (!attrs) return 0; - Bindings::iterator a = attrs->find(state->sMeta); - if (a == attrs->end()) return 0; - state->forceAttrs(*a->value, *a->pos); - meta = a->value->attrs; + auto val = queryOptionalAttr(state->sMeta); + if (!val) return 0; + state->forceAttrs(*val); + meta = val->attrs; return meta; } @@ -285,7 +302,7 @@ static bool getDerivation(EvalState & state, Value & v, derivation {...}; y = x;}'. */ if (!done.insert(v.attrs).second) return false; - DrvInfo drv(state, attrPath, v.attrs); + DrvInfo drv(state, attrPath, v); drv.queryName(); diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 29bb6a660..5f8c5661b 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -26,7 +26,8 @@ private: bool failed = false; // set if we get an AssertionError - Bindings * attrs = nullptr, * meta = nullptr; + std::optional attrs; + Bindings * meta = nullptr; Bindings * getMeta(); @@ -36,9 +37,12 @@ public: string attrPath; /* path towards the derivation */ DrvInfo(EvalState & state) : state(&state) { }; - DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs); + DrvInfo(EvalState & state, const string & attrPath, std::optional attrs); DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs); + Value queryAttr(const Symbol attrName) const; + std::optional queryOptionalAttr(const Symbol attrName) const; + string queryName() const; string querySystem() const; string queryDrvPath() const; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index cbe9a45bf..2a85365b4 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -166,6 +166,10 @@ struct ExprSelect : Expr AttrPath attrPath; ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; + ExprSelect(const Pos & pos, Expr * e, const std::vector & names) : pos(pos), e(e), def(0) { + for (auto & name : names) + attrPath.push_back(AttrName(name)); + }; COMMON_METHODS }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1d1afa768..94e7dd5c6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -410,7 +410,7 @@ static RegisterPrimOp primop_isNull({ Return `true` if *e* evaluates to `null`, and `false` otherwise. > **Warning** - > + > > This function is *deprecated*; just write `e == null` instead. )", .fun = prim_isNull, @@ -813,17 +813,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * { state.forceAttrs(*args[0], pos); + Value & arg = *(args[0]); + + auto getOptionalAttr= [&](Symbol attrName) -> std::optional { + auto accessResult = state.getOptionalAttrField(arg, {attrName}, pos); + return accessResult.error ? std::nullopt : std::optional{*accessResult.value}; + }; + /* Figure out the name first (for stack backtraces). */ - Bindings::iterator attr = args[0]->attrs->find(state.sName); - if (attr == args[0]->attrs->end()) - throw EvalError({ - .msg = hintfmt("required attribute 'name' missing"), - .errPos = pos - }); + state.getAttrField(arg, {state.sName}, pos, v); string drvName; - Pos & posDrvName(*attr->pos); + auto posDrvName = pos; // FIXME: Should be tho position of the `name` attribute try { - drvName = state.forceStringNoCtx(*attr->value, pos); + drvName = state.forceStringNoCtx(v, pos); } catch (Error & e) { e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'"); throw; @@ -832,15 +834,15 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Check whether attributes should be passed as a JSON file. */ std::ostringstream jsonBuf; std::unique_ptr jsonObject; - attr = args[0]->attrs->find(state.sStructuredAttrs); - if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) + if (auto maybeRawObj = getOptionalAttr(state.sStructuredAttrs)) { + if (state.forceBool(*maybeRawObj, pos)) jsonObject = std::make_unique(jsonBuf); + } /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; - attr = args[0]->attrs->find(state.sIgnoreNulls); - if (attr != args[0]->attrs->end()) - ignoreNulls = state.forceBool(*attr->value, pos); + if (auto maybeRawIgnoreNull = getOptionalAttr(state.sIgnoreNulls)) + ignoreNulls = state.forceBool(*maybeRawIgnoreNull, pos); /* Build the derivation expression by processing the attributes. */ Derivation drv; @@ -856,9 +858,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * StringSet outputs; outputs.insert("out"); - for (auto & i : args[0]->attrs->lexicographicOrder()) { - if (i->name == state.sIgnoreNulls) continue; - const string & key = i->name; + for (auto & currentArg : args[0]->attrs->lexicographicOrder()) { + if (currentArg->name == state.sIgnoreNulls) continue; + const string & key = currentArg->name; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string & s) { @@ -900,22 +902,26 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * try { + Value argValue; + state.getAttrField(arg, {currentArg->name}, pos, argValue); + + if (ignoreNulls) { - state.forceValue(*i->value, pos); - if (i->value->type() == nNull) continue; + state.forceValue(argValue, pos); + if (argValue.type() == nNull) continue; } - if (i->name == state.sContentAddressed) { + if (currentArg->name == state.sContentAddressed) { settings.requireExperimentalFeature("ca-derivations"); - contentAddressed = state.forceBool(*i->value, pos); + contentAddressed = state.forceBool(argValue, pos); } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ - else if (i->name == state.sArgs) { - state.forceList(*i->value, pos); - for (unsigned int n = 0; n < i->value->listSize(); ++n) { - string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); + else if (currentArg->name == state.sArgs) { + state.forceList(argValue, pos); + for (unsigned int n = 0; n < argValue.listSize(); ++n) { + string s = state.coerceToString(posDrvName, *argValue.listElems()[n], context, true); drv.args.push_back(s); } } @@ -926,39 +932,39 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (jsonObject) { - if (i->name == state.sStructuredAttrs) continue; + if (currentArg->name == state.sStructuredAttrs) continue; auto placeholder(jsonObject->placeholder(key)); - printValueAsJSON(state, true, *i->value, placeholder, context); + printValueAsJSON(state, true, argValue, placeholder, context); - if (i->name == state.sBuilder) - drv.builder = state.forceString(*i->value, context, posDrvName); - else if (i->name == state.sSystem) - drv.platform = state.forceStringNoCtx(*i->value, posDrvName); - else if (i->name == state.sOutputHash) - outputHash = state.forceStringNoCtx(*i->value, posDrvName); - else if (i->name == state.sOutputHashAlgo) - outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); - else if (i->name == state.sOutputHashMode) - handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); - else if (i->name == state.sOutputs) { + if (currentArg->name == state.sBuilder) + drv.builder = state.forceString(argValue, context, posDrvName); + else if (currentArg->name == state.sSystem) + drv.platform = state.forceStringNoCtx(argValue, posDrvName); + else if (currentArg->name == state.sOutputHash) + outputHash = state.forceStringNoCtx(argValue, posDrvName); + else if (currentArg->name == state.sOutputHashAlgo) + outputHashAlgo = state.forceStringNoCtx(argValue, posDrvName); + else if (currentArg->name == state.sOutputHashMode) + handleHashMode(state.forceStringNoCtx(argValue, posDrvName)); + else if (currentArg->name == state.sOutputs) { /* Require ‘outputs’ to be a list of strings. */ - state.forceList(*i->value, posDrvName); + state.forceList(argValue, posDrvName); Strings ss; - for (unsigned int n = 0; n < i->value->listSize(); ++n) - ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); + for (unsigned int n = 0; n < argValue.listSize(); ++n) + ss.emplace_back(state.forceStringNoCtx(*argValue.listElems()[n], posDrvName)); handleOutputs(ss); } } else { - auto s = state.coerceToString(posDrvName, *i->value, context, true); + auto s = state.coerceToString(posDrvName, argValue, context, true); drv.env.emplace(key, s); - if (i->name == state.sBuilder) drv.builder = s; - else if (i->name == state.sSystem) drv.platform = s; - else if (i->name == state.sOutputHash) outputHash = s; - else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s; - else if (i->name == state.sOutputHashMode) handleHashMode(s); - else if (i->name == state.sOutputs) + if (currentArg->name == state.sBuilder) drv.builder = s; + else if (currentArg->name == state.sSystem) drv.platform = s; + else if (currentArg->name == state.sOutputHash) outputHash = s; + else if (currentArg->name == state.sOutputHashAlgo) outputHashAlgo = s; + else if (currentArg->name == state.sOutputHashMode) handleHashMode(s); + else if (currentArg->name == state.sOutputs) handleOutputs(tokenizeString(s)); } @@ -1918,26 +1924,26 @@ static RegisterPrimOp primop_path({ An enrichment of the built-in path type, based on the attributes present in *args*. All are optional except `path`: - - path + - path The underlying path. - - name + - name The name of the path when added to the store. This can used to reference paths that have nix-illegal characters in their names, like `@`. - - filter + - filter A function of the type expected by `builtins.filterSource`, with the same semantics. - - recursive + - recursive When `false`, when `path` is added to the store it is with a flat hash, rather than a hash of the NAR serialization of the file. Thus, `path` must refer to a regular file, not a directory. This allows similar behavior to `fetchurl`. Defaults to `true`. - - sha256 + - sha256 When provided, this is the expected hash of the file at the path. Evaluation will fail if the hash is incorrect, and providing a hash allows `builtins.path` to be used even when the @@ -1956,13 +1962,14 @@ static RegisterPrimOp primop_path({ strings. */ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, Value & v) { + auto attrNames = state.listAttrFields(*args[0], pos); state.forceAttrs(*args[0], pos); - state.mkList(v, args[0]->attrs->size()); + state.mkList(v, attrNames.size()); size_t n = 0; - for (auto & i : *args[0]->attrs) - mkString(*(v.listElems()[n++] = state.allocValue()), i.name); + for (auto & name : attrNames) + mkString(*(v.listElems()[n++] = state.allocValue()), name); std::sort(v.listElems(), v.listElems() + n, [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); @@ -2013,17 +2020,8 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) { string attr = state.forceStringNoCtx(*args[0], pos); state.forceAttrs(*args[1], pos); + state.getAttrField(*args[1], {state.symbols.create(attr)}, pos, v); // !!! Should we create a symbol here or just do a lookup? - Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); - if (i == args[1]->attrs->end()) - throw EvalError({ - .msg = hintfmt("attribute '%1%' missing", attr), - .errPos = pos - }); - // !!! add to stack trace? - if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; - state.forceValue(*i->value, pos); - v = *i->value; } static RegisterPrimOp primop_getAttr({ @@ -2356,7 +2354,13 @@ static RegisterPrimOp primop_isList({ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v) { + auto evalCache = list.getCache(); + auto atSymbol = state.symbols.create(std::to_string(n)); + auto cacheResult = evalCache.getValue(*state.store, {atSymbol}, v); + if (cacheResult.returnCode == ValueCache::CacheHit) return; + state.forceList(list, pos); + evalCache.addListChilds(state.symbols, list.listElems(), list.listSize()); if (n < 0 || (unsigned int) n >= list.listSize()) throw Error({ .msg = hintfmt("list index %1% is out of bounds", n), @@ -2424,7 +2428,7 @@ static RegisterPrimOp primop_tail({ the argument isn’t a list or is an empty list. > **Warning** - > + > > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. diff --git a/src/libexpr/tree-cache.cc b/src/libexpr/tree-cache.cc new file mode 100644 index 000000000..e051cab7d --- /dev/null +++ b/src/libexpr/tree-cache.cc @@ -0,0 +1,397 @@ +#include "tree-cache.hh" +#include "sqlite.hh" +#include "store-api.hh" +#include "context.hh" + +namespace nix::tree_cache { + +static const char * schema = R"sql( +create table if not exists Attributes ( + id integer primary key autoincrement not null, + parent integer not null, + name text, + type integer not null, + value text, + context text, + unique (parent, name) +); + +create index if not exists IndexByParent on Attributes(parent, name); +)sql"; + +struct AttrDb +{ + std::atomic_bool failed{false}; + + struct State + { + SQLite db; + SQLiteStmt insertAttribute; + SQLiteStmt updateAttribute; + SQLiteStmt insertAttributeWithContext; + SQLiteStmt queryAttribute; + SQLiteStmt queryAttributes; + std::unique_ptr txn; + }; + + std::unique_ptr> _state; + + AttrDb(const Hash & fingerprint) + : _state(std::make_unique>()) + { + auto state(_state->lock()); + + Path cacheDir = getCacheDir() + "/nix/eval-cache-v3"; + createDirs(cacheDir); + + Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->insertAttribute.create(state->db, + "insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); + + state->updateAttribute.create(state->db, + "update Attributes set type = ?, value = ?, context = ? where id = ?"); + + state->insertAttributeWithContext.create(state->db, + "insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)"); + + state->queryAttribute.create(state->db, + "select id, type, value, context from Attributes where parent = ? and name = ?"); + + state->queryAttributes.create(state->db, + "select name from Attributes where parent = ?"); + + state->txn = std::make_unique(state->db); + } + + ~AttrDb() + { + try { + auto state(_state->lock()); + if (!failed) + state->txn->commit(); + state->txn.reset(); + } catch (...) { + ignoreException(); + } + } + + template + AttrId doSQLite(F && fun) + { + if (failed) return 0; + try { + return fun(); + } catch (SQLiteError &) { + ignoreException(); + failed = true; + return 0; + } + } + + /** + * Store a leaf of the tree in the db + */ + AttrId addEntry( + const AttrKey & key, + const AttrValue & value) + { + return doSQLite([&]() + { + auto state(_state->lock()); + auto rawValue = RawValue::fromVariant(value); + + state->insertAttributeWithContext.use() + (key.first) + (key.second) + (rawValue.type) + (rawValue.value.value_or(""), rawValue.value.has_value()) + (rawValue.serializeContext()) + .exec(); + AttrId rowId = state->db.getLastInsertedRowId(); + assert(rowId); + return rowId; + }); + } + + AttrId setBool( + AttrKey key, + bool b) + { + return addEntry(key, b); + } + + std::optional getId(const AttrKey& key) + { + auto state(_state->lock()); + + auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); + if (!queryAttribute.next()) return std::nullopt; + + return (AttrType) queryAttribute.getInt(0); + } + + AttrId setOrUpdate(const AttrKey& key, const AttrValue& value) + { + debug("cache: miss for the attribute %s", key.second); + if (auto existingId = getId(key)) { + setValue(*existingId, value); + return *existingId; + } + return addEntry(key, value); + } + + void setValue(const AttrId & id, const AttrValue & value) + { + auto state(_state->lock()); + auto rawValue = RawValue::fromVariant(value); + + state->updateAttribute.use() + (rawValue.type) + (rawValue.value.value_or(""), rawValue.value.has_value()) + (rawValue.serializeContext()) + (id) + .exec(); + } + + std::optional> getValue(AttrKey key) + { + auto state(_state->lock()); + + auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); + if (!queryAttribute.next()) return {}; + + auto rowId = (AttrType) queryAttribute.getInt(0); + auto type = (AttrType) queryAttribute.getInt(1); + + switch (type) { + case AttrType::Attrs: { + return {{rowId, attributeSet_t()}}; + } + case AttrType::String: { + std::vector> context; + if (!queryAttribute.isNull(3)) + for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) + context.push_back(decodeContext(s)); + return {{rowId, string_t{queryAttribute.getStr(2), context}}}; + } + case AttrType::Bool: + return {{rowId, queryAttribute.getInt(2) != 0}}; + case AttrType::Int: + return {{rowId, queryAttribute.getInt(2)}}; + case AttrType::Double: + return {{rowId, queryAttribute.getInt(2)}}; + case AttrType::Unknown: + return {{rowId, unknown_t{}}}; + case AttrType::Thunk: + return {{rowId, thunk_t{}}}; + case AttrType::Missing: + return {{rowId, missing_t{key.second}}}; + case AttrType::Failed: + return {{rowId, failed_t{queryAttribute.getStr(2)}}}; + default: + throw Error("unexpected type in evaluation cache"); + } + } + + std::vector getChildren(AttrId parentId) + { + std::vector res; + auto state(_state->lock()); + + auto queryAttributes(state->queryAttributes.use()(parentId)); + + while (queryAttributes.next()) + res.push_back(queryAttributes.getStr(0)); + + return res; + } +}; + +Cache::Cache(const Hash & useCache, + SymbolTable & symbols) + : db(std::make_shared(useCache)) + , symbols(symbols) + , rootSymbol(symbols.create("")) +{ +} + +std::shared_ptr Cache::tryCreate(const Hash & useCache, SymbolTable & symbols) +{ + try { + return std::make_shared(useCache, symbols); + } catch (SQLiteError &) { + ignoreException(); + return nullptr; + } +} + +void Cache::commit() +{ + if (db) { + debug("Saving the cache"); + auto state(db->_state->lock()); + if (state->txn->active) { + state->txn->commit(); + state->txn.reset(); + state->txn = std::make_unique(state->db); + } + } +} + +Cursor::Ref Cache::getRoot() +{ + return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{}); +} + +Cursor::Cursor( + ref root, + const Parent & parent, + const AttrValue& value + ) + : root(root) + , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt) + , label(parent ? parent->second : root->rootSymbol) + , cachedValue({root->db->setOrUpdate(getKey(), value), value}) +{ +} + +Cursor::Cursor( + ref root, + const Parent & parent, + const AttrId & id, + const AttrValue & value + ) + : root(root) + , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt) + , label(parent ? parent->second : root->rootSymbol) + , cachedValue({id, value}) +{ +} + + +AttrKey Cursor::getKey() +{ + if (!parentId) + return {0, root->rootSymbol}; + return {*parentId, label}; +} + +AttrValue Cursor::getCachedValue() +{ + return cachedValue.second; +} + +void Cursor::setValue(const AttrValue & v) +{ + root->db->setValue(cachedValue.first, v); + cachedValue.second = v; +} + +Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v) +{ + Parent parent = {{*this, attrPath}}; + auto childCursor = new Cursor( + root, + parent, + v + ); + return childCursor; +} + +std::vector Cursor::getChildren() +{ + return root->db->getChildren(cachedValue.first); +} + +std::optional> Cursor::getChildrenAtPath(const std::vector & attrPath) +{ + auto cursorAtPath = findAlongAttrPath(attrPath); + if (cursorAtPath) + return cursorAtPath->getChildren(); + return std::nullopt; +} + +Cursor::Ref Cursor::maybeGetAttr(const Symbol & name) +{ + auto rawAttr = root->db->getValue({cachedValue.first, name}); + if (rawAttr) { + Parent parent = {{*this, name}}; + debug("cache: hit for the attribute %s", cachedValue.first); + return new Cursor ( + root, parent, rawAttr->first, + rawAttr->second); + } + if (std::holds_alternative(cachedValue.second)) { + // If the parent is an attribute set but we're not present in the db, + // then we're not a member of this attribute set. So mark as missing + return addChild(name, missing_t{name}); + } + return nullptr; +} + +Cursor::Ref Cursor::findAlongAttrPath(const std::vector & attrPath) +{ + auto currentCursor = this; + for (auto & currentAccessor : attrPath) { + currentCursor = currentCursor->maybeGetAttr(currentAccessor); + if (!currentCursor) + break; + if (std::holds_alternative(currentCursor->cachedValue.second)) + break; + if (std::holds_alternative(currentCursor->cachedValue.second)) + break; + } + return currentCursor; +} + +const RawValue RawValue::fromVariant(const AttrValue & value) +{ + RawValue res; + std::visit(overloaded{ + [&](attributeSet_t x) { res.type = AttrType::Attrs; }, + [&](string_t x) { + res.type = AttrType::String; + res.value = x.first; + res.context = x.second; + }, + [&](bool x) { + res.type = AttrType::Bool; + res.value = x ? "1" : "0"; + }, + [&](int64_t x) { + res.type = AttrType::Int; + res.value = std::to_string(x); + }, + [&](double x) { + res.type = AttrType::Double; + res.value = std::to_string(x); + }, + [&](unknown_t x) { res.type = AttrType::Unknown; }, + [&](missing_t x) { res.type = AttrType::Missing; }, + [&](thunk_t x) { res.type = AttrType::Thunk; }, + [&](failed_t x) { + res.type = AttrType::Failed; + res.value = x.error; + } + }, value); + return res; +} + +std::string RawValue::serializeContext() const +{ + std::string res; + for (auto & elt : context) { + res.append(encodeContext(elt.second, elt.first)); + res.push_back(' '); + } + if (!res.empty()) + res.pop_back(); // Remove the trailing space + return res; +} + +} diff --git a/src/libexpr/tree-cache.hh b/src/libexpr/tree-cache.hh new file mode 100644 index 000000000..c4b7f06ea --- /dev/null +++ b/src/libexpr/tree-cache.hh @@ -0,0 +1,148 @@ +/** + * caching for a tree-like data structure (like Nix values) + * + * The cache is an sqlite db whose rows are the nodes of the tree, with a + * pointer to their parent (except for the root of course) + */ + +#pragma once + +#include "sync.hh" +#include "hash.hh" +#include "symbol-table.hh" + +#include +#include + +namespace nix::tree_cache { + +struct AttrDb; +class Cursor; + +class Cache : public std::enable_shared_from_this +{ +private: + friend class Cursor; + + /** + * The database holding the cache + */ + std::shared_ptr db; + + SymbolTable & symbols; + + /** + * Distinguished symbol indicating the root of the tree + */ + const Symbol rootSymbol; + +public: + + Cache( + const Hash & useCache, + SymbolTable & symbols + ); + + static std::shared_ptr tryCreate(const Hash & useCache, SymbolTable & symbols); + + Cursor * getRoot(); + + /** + * Flush the cache to disk + */ + void commit(); +}; + +enum AttrType { + Unknown = 0, + Attrs = 1, + String = 2, + Bool = 3, + Int = 4, + Double = 5, + Thunk = 6, + Missing = 7, // Missing fields of attribute sets + Failed = 8, +}; + +struct attributeSet_t {}; +struct unknown_t {}; +struct thunk_t {}; +struct failed_t { string error; }; +struct missing_t { Symbol attrName; }; +typedef uint64_t AttrId; + +typedef std::pair AttrKey; +typedef std::pair>> string_t; + +typedef std::variant< + attributeSet_t, + string_t, + unknown_t, + thunk_t, + missing_t, + failed_t, + bool, + int64_t, + double +> AttrValue; + +struct RawValue { + AttrType type; + std::optional value; + std::vector> context; + + std::string serializeContext() const; + + static const RawValue fromVariant(const AttrValue&); + AttrValue toVariant() const; +}; + +/** + * View inside the cache. + * + * A `Cursor` represents a node in the cached tree (be it a leaf or not) + */ +class Cursor : public std::enable_shared_from_this +{ + /** + * The overall cache of which this cursor is a view + */ + ref root; + + typedef std::optional> Parent; + + std::optional parentId; + Symbol label; + + std::pair cachedValue; + + /** + * Get the identifier for this node in the database + */ + AttrKey getKey(); + +public: + + using Ref = Cursor*; + + // Create a new cache entry + Cursor(ref root, const Parent & parent, const AttrValue&); + // Build a cursor from an existing cache entry + Cursor(ref root, const Parent & parent, const AttrId& id, const AttrValue&); + + AttrValue getCachedValue(); + + void setValue(const AttrValue & v); + + Ref addChild(const Symbol & attrPath, const AttrValue & v); + + Ref findAlongAttrPath(const std::vector & attrPath); + Ref maybeGetAttr(const Symbol & attrPath); + + + std::vector getChildren(); + std::optional> getChildrenAtPath(const std::vector & attrPath); +}; + +} diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index b317c1898..ee25de264 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,6 +1,7 @@ #pragma once #include "symbol-table.hh" +#include "tree-cache.hh" #if HAVE_BOEHMGC #include @@ -56,8 +57,51 @@ struct Pos; class EvalState; class XMLWriter; class JSONPlaceholder; +class Store; +struct Value; +class ValueCache { + tree_cache::Cursor::Ref rawCache; + +public: + + ValueCache(tree_cache::Cursor::Ref rawCache) : rawCache(rawCache) {} + + const static ValueCache empty; + + enum ReturnCode { + // The cache result was an attribute set, so we forward it later in the + // chain + Forward, + CacheMiss, + CacheHit, + UnCacheable, + NoCacheKey, + }; + + struct CacheResult { + ReturnCode returnCode; + + // In case the query returns a `missing_t`, the symbol that's missing + std::optional lastQueriedSymbolIfMissing; + }; + CacheResult getValue(Store & store, const std::vector & selector, Value & dest); + + ValueCache addChild(const Symbol & attrName, const Value & value); + ValueCache addFailedChild(const Symbol & attrName, const Error & error); + ValueCache addNumChild(SymbolTable & symbols, int idx, const Value & value); + void addAttrSetChilds(Bindings & children); + void addListChilds(SymbolTable & symbols, Value** elems, int listSize); + + std::optional> listChildren(SymbolTable&); + std::optional> listChildrenAtPath(SymbolTable&, const std::vector & attrPath); + + std::optional getRawValue(); + + ValueCache() : rawCache(nullptr) {} +}; + typedef int64_t NixInt; typedef double NixFloat; @@ -113,6 +157,12 @@ friend std::string showType(const Value & v); friend void printValue(std::ostream & str, std::set & active, const Value & v); public: + /* + * An optional evaluation cache (for flakes in particular). + * If this is set, then trying to get a value from this attrset will first + * try to get it from the cache + */ + ValueCache evalCache; // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's @@ -346,7 +396,19 @@ public: non-trivial. */ bool isTrivial() const; - std::vector> getContext(); + std::vector> getContext() const; + + /* + * Set the associated cache view for this value. + * This cache will be used to speed-up some operations like accessing + * attribute sets elements. + */ + void setCache(ValueCache); + + /* + * Get the cache associated with this value, if any. + */ + ValueCache getCache() const; }; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 106a78fc4..1585214a9 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -374,7 +374,7 @@ static void queryInstSources(EvalState & state, std::string name(path.name()); - DrvInfo elem(state, "", nullptr); + DrvInfo elem(state, "", std::nullopt); elem.setName(name); if (path.isDerivation()) { diff --git a/src/nix/app.cc b/src/nix/app.cc index 80acbf658..89f798698 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -1,26 +1,28 @@ #include "installables.hh" #include "store-api.hh" #include "eval-inline.hh" -#include "eval-cache.hh" #include "names.hh" namespace nix { App Installable::toApp(EvalState & state) { - auto [cursor, attrPath] = getCursor(state); + auto [value, pos, attrPath] = toValue(state); - auto type = cursor->getAttr("type")->getString(); + auto type = state.forceStringNoCtx(*state.getAttrField(*value, {state.sType}, pos)); if (type == "app") { - auto [program, context] = cursor->getAttr("program")->getStringWithContext(); + StringSet context; + auto program = state.forceString(*state.getAttrField(*value, {state.symbols.create("program")}, pos)); if (!state.store->isInStore(program)) throw Error("app program '%s' is not in the Nix store", program); std::vector context2; - for (auto & [path, name] : context) + for (auto & rawCtxItem : context) { + auto [path, name] = decodeContext(rawCtxItem); context2.push_back({state.store->parseStorePath(path), {name}}); + } return App { .context = std::move(context2), @@ -29,10 +31,10 @@ App Installable::toApp(EvalState & state) } else if (type == "derivation") { - auto drvPath = cursor->forceDerivation(); - auto outPath = cursor->getAttr(state.sOutPath)->getString(); - auto outputName = cursor->getAttr(state.sOutputName)->getString(); - auto name = cursor->getAttr(state.sName)->getString(); + auto drvPath = state.store->parseStorePath(state.forceString(*state.getAttrField(*value, {state.sDrvPath}, pos))); + auto outPath = state.forceString(*state.getAttrField(*value, {state.sOutPath}, pos)); + auto outputName = state.forceStringNoCtx(*state.getAttrField(*value, {state.sOutputName}, pos)); + auto name = state.forceStringNoCtx(*state.getAttrField(*value, {state.sName}, pos)); return App { .context = { { drvPath, {outputName} } }, .program = outPath + "/bin/" + DrvName(name).name, diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 1789e4598..b8fe144a0 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -92,7 +92,7 @@ struct CmdBundle : InstallableCommand arg->attrs->sort(); auto vRes = evalState->allocValue(); - evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); + evalState->callFunction(*bundler.toValue(*evalState).value, *arg, *vRes, noPos); if (!evalState->isDerivation(*vRes)) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 6472dd27a..f458f3935 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -28,7 +28,7 @@ struct CmdEdit : InstallableCommand { auto state = getEvalState(); - auto [v, pos] = installable->toValue(*state); + auto [v, pos, _] = installable->toValue(*state); try { pos = findDerivationFilename(*state, *v, installable->what()); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 65d61e005..93df1c2fa 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -60,7 +60,7 @@ struct CmdEval : MixJSON, InstallableCommand auto state = getEvalState(); - auto [v, pos] = installable->toValue(*state); + auto [v, pos, _] = installable->toValue(*state); PathSet context; if (apply) { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4cd7d77a0..a278011e3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -11,7 +11,6 @@ #include "fetchers.hh" #include "registry.hh" #include "json.hh" -#include "eval-cache.hh" #include #include @@ -600,9 +599,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand Strings{templateName == "" ? "defaultTemplate" : templateName}, Strings(attrsPathPrefixes), lockFlags); - auto [cursor, attrPath] = installable.getCursor(*evalState); - - auto templateDir = cursor->getAttr("path")->getString(); + auto flakeValue = installable.toValue(*evalState); + auto templateDir = evalState->forceStringNoCtx(*evalState->getAttrField(*flakeValue.value, {evalState->symbols.create("path")}, flakeValue.pos)); assert(store->isInStore(templateDir)); @@ -829,9 +827,9 @@ struct CmdFlakeShow : FlakeCommand auto state = getEvalState(); auto flake = std::make_shared(lockFlake()); - std::function & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit; + std::function & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit; - visit = [&](eval_cache::AttrCursor & visitor, const std::vector & attrPath, const std::string & headerPrefix, const std::string & nextPrefix) + visit = [&](Value & value, Pos & pos, const std::vector & attrPath, const std::string & headerPrefix, const std::string & nextPrefix) { Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", concatStringsSep(".", attrPath))); @@ -839,13 +837,13 @@ struct CmdFlakeShow : FlakeCommand auto recurse = [&]() { logger->cout("%s", headerPrefix); - auto attrs = visitor.getAttrs(); + auto attrs = state->listAttrFields(value, pos); for (const auto & [i, attr] : enumerate(attrs)) { bool last = i + 1 == attrs.size(); - auto visitor2 = visitor.getAttr(attr); + auto value2 = state->getAttrField(value, {attr}, pos); auto attrPath2(attrPath); attrPath2.push_back(attr); - visit(*visitor2, attrPath2, + visit(*value2, pos, attrPath2, fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr), nextPrefix + (last ? treeNull : treeLine)); } @@ -853,7 +851,7 @@ struct CmdFlakeShow : FlakeCommand auto showDerivation = [&]() { - auto name = visitor.getAttr(state->sName)->getString(); + auto name = state->forceStringNoCtx(*state->getAttrField(value, {state->sName}, pos)); /* std::string description; @@ -895,14 +893,14 @@ struct CmdFlakeShow : FlakeCommand || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages")) ) { - if (visitor.isDerivation()) + if (state->isDerivation(value)) showDerivation(); else throw Error("expected a derivation"); } else if (attrPath.size() > 0 && attrPath[0] == "hydraJobs") { - if (visitor.isDerivation()) + if (state->isDerivation(value)) showDerivation(); else recurse(); @@ -914,7 +912,7 @@ struct CmdFlakeShow : FlakeCommand else if (!showLegacy) logger->cout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix); else { - if (visitor.isDerivation()) + if (state->isDerivation(value)) showDerivation(); else if (attrPath.size() <= 2) // FIXME: handle recurseIntoAttrs @@ -926,8 +924,8 @@ struct CmdFlakeShow : FlakeCommand (attrPath.size() == 2 && attrPath[0] == "defaultApp") || (attrPath.size() == 3 && attrPath[0] == "apps")) { - auto aType = visitor.maybeGetAttr("type"); - if (!aType || aType->getString() != "app") + auto aType = state->getOptionalAttrField(value, {state->sType}, pos); + if (aType.error || state->forceStringNoCtx(*aType.value) != "app") throw EvalError("not an app definition"); logger->cout("%s: app", headerPrefix); } @@ -936,7 +934,8 @@ struct CmdFlakeShow : FlakeCommand (attrPath.size() == 1 && attrPath[0] == "defaultTemplate") || (attrPath.size() == 2 && attrPath[0] == "templates")) { - auto description = visitor.getAttr("description")->getString(); + /* auto description = visitor.getAttr("description")->getString(); */ + auto description = state->forceStringNoCtx(*state->getAttrField(value, {state->sDescription}, pos)); logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); } @@ -954,9 +953,9 @@ struct CmdFlakeShow : FlakeCommand } }; - auto cache = openEvalCache(*state, flake); + auto flakeValue = getFlakeValue(*state, *flake); - visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); + visit(*flakeValue, noPos, {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); } }; diff --git a/src/nix/search.cc b/src/nix/search.cc index c52a48d4e..a5c92df0a 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -7,7 +7,6 @@ #include "common-args.hh" #include "json.hh" #include "shared.hh" -#include "eval-cache.hh" #include "attr-path.hh" #include @@ -81,31 +80,38 @@ struct CmdSearch : InstallableCommand, MixJSON uint64_t results = 0; - std::function & attrPath, bool initialRecurse)> visit; + std::function & attrPath, bool initialRecurse)> visit; - visit = [&](eval_cache::AttrCursor & cursor, const std::vector & attrPath, bool initialRecurse) + visit = [&](Value & cursor, const Pos& pos, const std::vector & attrPath, bool initialRecurse) { Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", concatStringsSep(".", attrPath))); try { auto recurse = [&]() { - for (const auto & attr : cursor.getAttrs()) { - auto cursor2 = cursor.getAttr(attr); + for (const auto & attr : evalState->listAttrFields(cursor, pos)) { + Value* cursor2; + try { + auto cursor2Res = evalState->lazyGetOptionalAttrField(cursor, {attr}, pos); + if (cursor2Res.error) + continue; // Shouldn't happen, but who knows + cursor2 = cursor2Res.value; + } catch (EvalError&) { + continue; + } auto attrPath2(attrPath); attrPath2.push_back(attr); - visit(*cursor2, attrPath2, false); + visit(*cursor2, pos, attrPath2, false); } }; - if (cursor.isDerivation()) { + if (evalState->isDerivation(cursor)) { size_t found = 0; - DrvName name(cursor.getAttr("name")->getString()); + DrvName name(evalState->forceStringNoCtx(*evalState->getAttrField(cursor, {evalState->sName}, pos))); - auto aMeta = cursor.maybeGetAttr("meta"); - auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr; - auto description = aDescription ? aDescription->getString() : ""; + auto descriptionGetRes = evalState->getOptionalAttrField(cursor, {evalState->sMeta, evalState->sDescription}, pos); + auto description = !descriptionGetRes.error ? state->forceStringNoCtx(*descriptionGetRes.value) : ""; std::replace(description.begin(), description.end(), '\n', ' '); auto attrPath2 = concatStringsSep(".", attrPath); @@ -154,8 +160,8 @@ struct CmdSearch : InstallableCommand, MixJSON recurse(); else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) { - auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); - if (attr && attr->getBool()) + auto recurseForDrvGetRes = evalState->getOptionalAttrField(cursor, {evalState->sRecurseForDerivations}, pos); + if (!recurseForDrvGetRes.error && evalState->forceBool(*recurseForDrvGetRes.value, pos)) recurse(); } @@ -165,8 +171,8 @@ struct CmdSearch : InstallableCommand, MixJSON } }; - for (auto & [cursor, prefix] : installable->getCursors(*state)) - visit(*cursor, parseAttrPath(*state, prefix), true); + for (auto & [cursor, pos, prefix] : installable->toValues(*state)) + visit(*cursor, pos, parseAttrPath(*state, prefix), true); if (!json && !results) throw Error("no results for the given search term(s)!"); diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk index 82ad99402..e5c7b2f57 100644 --- a/tests/plugins/local.mk +++ b/tests/plugins/local.mk @@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 -libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr +libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr -I src/libstore