From 91313011f9f3d02dbdbafca56ce611b6925b3d8a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Mar 2008 15:29:03 +0000 Subject: [PATCH] * Conversion to a BDB-free store. Meta-information for each valid path (content hash, references, deriver) is now stored in /nix/var/nix/db/meta/.info. The referrers are stored separately in /nix/var/nix/db/meta/.referrers. This is done because the referrers can change, which the info file is in principle immutable. The referrers must also be updated efficiently to prevent a regression of NIX-23 (quadratic complexity updating the referrers during garbage collection). This is just the conversion code, the rest of the code still uses BDB. --- src/libstore/local-store.cc | 100 +++++++++++++++++++++++++++++++----- src/libstore/local-store.hh | 4 +- 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6a8de7bc4..e2a6421a1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -16,6 +16,7 @@ #include #include #include +#include // !!! remove namespace nix { @@ -58,6 +59,7 @@ static TableId dbDerivers = 0; static void upgradeStore09(); static void upgradeStore11(); +static void upgradeStore12(); void checkStoreNotSymlink() @@ -133,6 +135,7 @@ LocalStore::LocalStore(bool reserveSpace) throw Error("your Nix store is no longer supported"); if (curSchema <= 2) upgradeStore09(); if (curSchema <= 3) upgradeStore11(); + if (curSchema <= 4) upgradeStore12(); writeFile(schemaFN, (format("%1%") % nixSchemaVersion).str()); } } @@ -328,7 +331,7 @@ void setReferences(const Transaction & txn, const Path & storePath, } -void queryReferences(const Transaction & txn, +void oldQueryReferences(const Transaction & txn, const Path & storePath, PathSet & references) { Paths references2; @@ -342,7 +345,7 @@ void queryReferences(const Transaction & txn, void LocalStore::queryReferences(const Path & storePath, PathSet & references) { - nix::queryReferences(noTxn, storePath, references); + oldQueryReferences(noTxn, storePath, references); } @@ -375,7 +378,7 @@ void setDeriver(const Transaction & txn, const Path & storePath, } -static Path queryDeriver(const Transaction & txn, const Path & storePath) +static Path oldQueryDeriver(const Transaction & txn, const Path & storePath) { if (!isValidPathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); @@ -389,7 +392,7 @@ static Path queryDeriver(const Transaction & txn, const Path & storePath) Path LocalStore::queryDeriver(const Path & path) { - return nix::queryDeriver(noTxn, path); + return oldQueryDeriver(noTxn, path); } @@ -423,7 +426,7 @@ bool LocalStore::hasSubstitutes(const Path & path) } -static void setHash(const Transaction & txn, const Path & storePath, +static void oldSetHash(const Transaction & txn, const Path & storePath, const Hash & hash) { assert(hash.type == htSHA256); @@ -431,7 +434,7 @@ static void setHash(const Transaction & txn, const Path & storePath, } -static Hash queryHash(const Transaction & txn, const Path & storePath) +static Hash oldQueryHash(const Transaction & txn, const Path & storePath) { string s; nixDB.queryString(txn, dbValidPaths, storePath, s); @@ -451,7 +454,7 @@ Hash LocalStore::queryPathHash(const Path & path) { if (!isValidPath(path)) throw Error(format("path `%1%' is not valid") % path); - return queryHash(noTxn, path); + return oldQueryHash(noTxn, path); } @@ -484,7 +487,7 @@ void registerValidPaths(const Transaction & txn, assertStorePath(i->path); debug(format("registering path `%1%'") % i->path); - setHash(txn, i->path, i->hash); + oldSetHash(txn, i->path, i->hash); setReferences(txn, i->path, i->references); @@ -834,7 +837,7 @@ void verifyStore(bool checkContents) } else { if (checkContents) { debug(format("checking contents of `%1%'") % *i); - Hash expected = queryHash(txn, *i); + Hash expected = oldQueryHash(txn, *i); Hash current = hashPath(expected.type, *i); if (current != expected) { printMsg(lvlError, format("path `%1%' was modified! " @@ -864,7 +867,7 @@ void verifyStore(bool checkContents) nixDB.delPair(txn, dbDerivers, *i); } else { - Path deriver = queryDeriver(txn, *i); + Path deriver = oldQueryDeriver(txn, *i); if (!isStorePath(deriver)) { printMsg(lvlError, format("removing corrupt deriver `%1%' for `%2%'") % deriver % *i); @@ -888,7 +891,7 @@ void verifyStore(bool checkContents) } else { PathSet references; - queryReferences(txn, *i, references); + oldQueryReferences(txn, *i, references); for (PathSet::iterator j = references.begin(); j != references.end(); ++j) { @@ -936,7 +939,7 @@ void verifyStore(bool checkContents) else { PathSet references; - queryReferences(txn, from, references); + oldQueryReferences(txn, from, references); if (find(references.begin(), references.end(), to) == references.end()) { printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'") % from % to); @@ -1144,4 +1147,77 @@ static void upgradeStore11() } +Path metaFileFor(const Path & path) +{ + string baseName = baseNameOf(path); + return (format("%1%/meta/%2%") % nixDBPath % baseName).str(); +} + + +/* !!! move to util.cc */ +void appendFile(const Path & path, const string & s) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_APPEND | O_CREAT, 0666); + if (fd == -1) + throw SysError(format("opening file `%1%'") % path); + writeFull(fd, (unsigned char *) s.c_str(), s.size()); +} + + +void newRegisterValidPath(const Path & path, + const Hash & hash, const PathSet & references, + const Path & deriver) +{ + Path infoFile = metaFileFor(path) + ".info"; + + string refs; + for (PathSet::const_iterator i = references.begin(); i != references.end(); ++i) { + if (!refs.empty()) refs += " "; + refs += *i; + + /* Update the referrer mapping for *i. This must be done + before the info file is written to maintain the invariant + that if `path' is a valid path, then all its references + have referrer mappings back to `path'. A " " is prefixed + to separate it from the previous entry. It's not suffixed + to deal with interrupted partial writes to this file. */ + Path referrersFile = metaFileFor(*i) + ".referrers"; + /* !!! locking */ + appendFile(referrersFile, " " + path); + } + + string info = (format( + "Hash: sha256:%1%\n" + "References: %2%\n" + "Deriver: %3%\n" + "Registered-At: %4%\n") + % printHash(hash) % refs % deriver % time(0)).str(); + + // !!! atomicity + writeFile(infoFile, info); +} + + +/* Upgrade from schema 4 (Nix 0.11) to schema 5 (Nix >= 0.12). The + old schema uses Berkeley DB, the new one stores store path + meta-information in files. */ +static void upgradeStore12() +{ + printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); + + Paths validPaths; + nixDB.enumTable(noTxn, dbValidPaths, validPaths); + + for (Paths::iterator i = validPaths.begin(); i != validPaths.end(); ++i) { + Hash hash = oldQueryHash(noTxn, *i); + PathSet references; oldQueryReferences(noTxn, *i, references); + Path deriver = oldQueryDeriver(noTxn, *i); + newRegisterValidPath(*i, hash, references, deriver); + std::cerr << "."; + } + + std::cerr << std::endl; +} + + } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9e6130013..7bc329b7a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -14,8 +14,8 @@ class Transaction; /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. */ -const int nixSchemaVersion = 4; + Version 4 is Nix 0.11. Version 5 is Nix 0.12*/ +const int nixSchemaVersion = 5; extern string drvsLogDir;