#include "tree-cache.hh" #include "sqlite.hh" #include "store-api.hh" #include "context.hh" namespace nix::tree_cache { static const char * schema = R"sql( create table if not exists Attributes ( id integer primary key autoincrement not null, parent integer not null, name text, type integer not null, value text, context text, unique (parent, name) ); create index if not exists IndexByParent on Attributes(parent, name); )sql"; struct AttrDb { std::atomic_bool failed{false}; struct State { SQLite db; SQLiteStmt insertAttribute; SQLiteStmt updateAttribute; SQLiteStmt insertAttributeWithContext; SQLiteStmt queryAttribute; SQLiteStmt queryAttributes; std::unique_ptr txn; }; std::unique_ptr> _state; AttrDb(const Hash & fingerprint) : _state(std::make_unique>()) { auto state(_state->lock()); Path cacheDir = getCacheDir() + "/nix/eval-cache-v3"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; state->db = SQLite(dbPath); state->db.isCache(); state->db.exec(schema); state->insertAttribute.create(state->db, "insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); state->updateAttribute.create(state->db, "update Attributes set type = ?, value = ?, context = ? where id = ?"); state->insertAttributeWithContext.create(state->db, "insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)"); state->queryAttribute.create(state->db, "select id, type, value, context from Attributes where parent = ? and name = ?"); state->queryAttributes.create(state->db, "select name, type from Attributes where parent = ?"); state->txn = std::make_unique(state->db); } ~AttrDb() { try { auto state(_state->lock()); if (!failed) state->txn->commit(); state->txn.reset(); } catch (...) { ignoreException(); } } template AttrId doSQLite(F && fun) { if (failed) return 0; try { return fun(); } catch (SQLiteError &) { ignoreException(); failed = true; return 0; } } /** * Store a leaf of the tree in the db */ AttrId addEntry( const AttrKey & key, const AttrValue & value) { return doSQLite([&]() { auto state(_state->lock()); auto rawValue = RawValue::fromVariant(value); state->insertAttributeWithContext.use() (key.first) (key.second) (rawValue.type) (rawValue.value.value_or(""), rawValue.value.has_value()) (rawValue.serializeContext()) .exec(); AttrId rowId = state->db.getLastInsertedRowId(); assert(rowId); return rowId; }); } std::optional getId(const AttrKey& key) { auto state(_state->lock()); auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); if (!queryAttribute.next()) return std::nullopt; return (AttrType) queryAttribute.getInt(0); } AttrId setOrUpdate(const AttrKey& key, const AttrValue& value) { debug("cache: miss for the attribute %s", key.second); if (auto existingId = getId(key)) { setValue(*existingId, value); return *existingId; } return addEntry(key, value); } void setValue(const AttrId & id, const AttrValue & value) { auto state(_state->lock()); auto rawValue = RawValue::fromVariant(value); state->updateAttribute.use() (rawValue.type) (rawValue.value.value_or(""), rawValue.value.has_value()) (rawValue.serializeContext()) (id) .exec(); } std::optional> getValue(AttrKey key) { auto state(_state->lock()); auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); if (!queryAttribute.next()) return {}; auto rowId = (AttrType) queryAttribute.getInt(0); auto type = (AttrType) queryAttribute.getInt(1); switch (type) { case AttrType::Attrs: { return {{rowId, attributeSet_t()}}; } case AttrType::String: { std::vector> context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) context.push_back(decodeContext(s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: return {{rowId, wrapped_basetype{queryAttribute.getInt(2) != 0}}}; case AttrType::Int: return {{rowId, wrapped_basetype{queryAttribute.getInt(2)}}}; case AttrType::Double: return {{rowId, wrapped_basetype{(double)queryAttribute.getInt(2)}}}; case AttrType::Unknown: return {{rowId, unknown_t{}}}; case AttrType::Thunk: return {{rowId, thunk_t{}}}; case AttrType::Missing: return {{rowId, missing_t{key.second}}}; case AttrType::Failed: return {{rowId, failed_t{queryAttribute.getStr(2)}}}; default: throw Error("unexpected type in evaluation cache"); } } std::vector getChildren(AttrId parentId) { std::vector res; auto state(_state->lock()); auto queryAttributes(state->queryAttributes.use()(parentId)); while (queryAttributes.next()) { if (queryAttributes.getInt(1) != AttrType::Missing) res.push_back(queryAttributes.getStr(0)); } return res; } }; Cache::Cache(const Hash & useCache, SymbolTable & symbols) : db(std::make_shared(useCache)) , symbols(symbols) , rootSymbol(symbols.create("%root%")) { } std::shared_ptr Cache::tryCreate(const Hash & useCache, SymbolTable & symbols) { try { return std::make_shared(useCache, symbols); } catch (SQLiteError &) { ignoreException(); return nullptr; } } void Cache::commit() { if (db) { debug("Saving the cache"); auto state(db->_state->lock()); if (state->txn->active) { state->txn->commit(); state->txn.reset(); state->txn = std::make_unique(state->db); } } } Cursor::Ref Cache::getRoot() { return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{}); } Cursor::Cursor( ref root, const Parent & parent, const AttrValue& value ) : root(root) , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt) , label(parent ? parent->second : root->rootSymbol) , cachedValue({root->db->setOrUpdate(getKey(), value), value}) { } Cursor::Cursor( ref root, const Parent & parent, const AttrId & id, const AttrValue & value ) : root(root) , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt) , label(parent ? parent->second : root->rootSymbol) , cachedValue({id, value}) { } AttrKey Cursor::getKey() { if (!parentId) return {0, root->rootSymbol}; return {*parentId, label}; } AttrValue Cursor::getCachedValue() { return cachedValue.second; } void Cursor::setValue(const AttrValue & v) { root->db->setValue(cachedValue.first, v); cachedValue.second = v; } Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v) { Parent parent = {{*this, attrPath}}; auto childCursor = new Cursor( root, parent, v ); return childCursor; } std::vector Cursor::getChildren() { return root->db->getChildren(cachedValue.first); } std::optional> Cursor::getChildrenAtPath(const std::vector & attrPath) { auto cursorAtPath = findAlongAttrPath(attrPath); if (cursorAtPath) return cursorAtPath->getChildren(); return std::nullopt; } Cursor::Ref Cursor::maybeGetAttr(const Symbol & name) { auto rawAttr = root->db->getValue({cachedValue.first, name}); if (rawAttr) { Parent parent = {{*this, name}}; debug("cache: hit for the attribute %s", cachedValue.first); return new Cursor ( root, parent, rawAttr->first, rawAttr->second); } if (std::holds_alternative(cachedValue.second)) { // If the parent is an attribute set but we're not present in the db, // then we're not a member of this attribute set. So mark as missing return addChild(name, missing_t{name}); } return nullptr; } Cursor::Ref Cursor::findAlongAttrPath(const std::vector & attrPath) { auto currentCursor = this; for (auto & currentAccessor : attrPath) { currentCursor = currentCursor->maybeGetAttr(currentAccessor); if (!currentCursor) break; if (std::holds_alternative(currentCursor->cachedValue.second)) break; if (std::holds_alternative(currentCursor->cachedValue.second)) break; } return currentCursor; } const RawValue RawValue::fromVariant(const AttrValue & value) { RawValue res; std::visit(overloaded{ [&](attributeSet_t x) { res.type = AttrType::Attrs; }, [&](string_t x) { res.type = AttrType::String; res.value = x.first; res.context = x.second; }, [&](wrapped_basetype x) { res.type = AttrType::Bool; res.value = x.value ? "1" : "0"; }, [&](wrapped_basetype x) { res.type = AttrType::Int; res.value = std::to_string(x.value); }, [&](wrapped_basetype x) { res.type = AttrType::Double; res.value = std::to_string(x.value); }, [&](unknown_t x) { res.type = AttrType::Unknown; }, [&](missing_t x) { res.type = AttrType::Missing; }, [&](thunk_t x) { res.type = AttrType::Thunk; }, [&](failed_t x) { res.type = AttrType::Failed; res.value = x.error; } }, value); return res; } std::string RawValue::serializeContext() const { std::string res; for (auto & elt : context) { res.append(encodeContext(elt.second, elt.first)); res.push_back(' '); } if (!res.empty()) res.pop_back(); // Remove the trailing space return res; } }