From b1cc9e9a45dd1d42f6d3c5b22ec4103a09e5b9cc Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Tue, 19 Jun 2007 13:04:05 +0000 Subject: [PATCH] Before moving scanForStateReferences(...) --- src/libstore/build.cc | 18 ++++- src/libstore/local-store.cc | 138 ++++++++++++++++---------------- src/libstore/local-store.hh | 11 +-- src/libstore/remote-store.cc | 18 ++++- src/libstore/remote-store.hh | 4 +- src/libstore/store-api.hh | 6 +- src/libstore/worker-protocol.hh | 2 + src/nix-state/nix-state.cc | 11 ++- src/nix-store/help.txt | 1 + src/nix-store/nix-store.cc | 11 ++- src/nix-worker/nix-worker.cc | 14 ++++ 11 files changed, 142 insertions(+), 92 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index bf32628ed..6c44d275b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -773,10 +773,9 @@ void DerivationGoal::haveDerivation() /* Check what outputs paths are not already valid. */ PathSet invalidOutputs = checkPathValidity(false); - //Just before we build, we resolve the multiple derivations linked to one store path issue, by choosing the latest derivation + //Just before we build the drvs, we already put in the database which component path is a state component path printMsg(lvlError, format("updateAllStateDerivations %1%") % drvPath); - //addStateDeriver(....); //only on succesfull! build - + store->registerMaybeStatePath(drvPath); /* If they are all valid, then we're done. */ if (invalidOutputs.size() == 0) { @@ -1696,13 +1695,24 @@ void DerivationGoal::computeClosure() PathSet allStatePaths; for (PathSet::const_iterator i = allPaths.begin(); i != allPaths.end(); i++){ Path componentPath = *i; + + printMsg(lvlError, format("COMP: %1%") % (*i)); + if(store->isStateComponent(componentPath)){ + printMsg(lvlError, format("COMP-STATE: %1%") % (*i)); + PathSet stateRefs = queryDerivers(noTxn, componentPath ,"*",getCallingUserName()); - stateRefs = mergePathSets(stateRefs, allStatePaths); + allStatePaths = mergePathSets(stateRefs, allStatePaths); } } PathSet stateReferences = scanForStateReferences(path, allStatePaths); + for (PathSet::const_iterator i = allStatePaths.begin(); i != allStatePaths.end(); i++){ + printMsg(lvlError, format("allStatePaths: %1%") % (*i)); + } + for (PathSet::const_iterator i = stateReferences.begin(); i != stateReferences.end(); i++){ + printMsg(lvlError, format("stateReferences: %1%") % (*i)); + } /* For debugging, print out the referenced and unreferenced paths. */ diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3077987b9..b4f3b67c6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -344,9 +344,8 @@ static PathSet getReferrers(const Transaction & txn, const Path & storePath) return referrers; } - void setReferences(const Transaction & txn, const Path & storePath, - const PathSet & references) + const PathSet & references, const PathSet & stateReferences) { /* For unrealisable paths, we can only clear the references. */ if (references.size() > 0 && !isRealisablePath(txn, storePath)) @@ -356,13 +355,22 @@ void setReferences(const Transaction & txn, const Path & storePath, Paths oldReferences; nixDB.queryStrings(txn, dbReferences, storePath, oldReferences); + PathSet oldReferences2(oldReferences.begin(), oldReferences.end()); - PathSet oldReferences2(oldReferences.begin(), oldReferences.end()); - if (oldReferences2 == references) return; + Paths oldStateReferences; + nixDB.queryStrings(txn, dbStateReferences, storePath, oldStateReferences); + PathSet oldStateReferences2(oldStateReferences.begin(), oldStateReferences.end()); + + if (oldReferences2 == references && oldStateReferences2 == stateReferences) return; nixDB.setStrings(txn, dbReferences, storePath, Paths(references.begin(), references.end())); + nixDB.setStrings(txn, dbStateReferences, storePath, + Paths(stateReferences.begin(), stateReferences.end())); + + //TODO THESE 2 ALSO FOR STATEREFS + /* Update the referrers mappings of all new referenced paths. */ for (PathSet::const_iterator i = references.begin(); i != references.end(); ++i) @@ -383,8 +391,6 @@ void queryReferences(const Transaction & txn, { Paths references2; - //WOUTER EDIT TODO - if (!isRealisablePath(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); nixDB.queryStrings(txn, dbReferences, storePath, references2); @@ -398,6 +404,25 @@ void LocalStore::queryReferences(const Path & storePath, nix::queryReferences(noTxn, storePath, references); } +void queryStateReferences(const Transaction & txn, + const Path & storePath, PathSet & stateReferences) +{ + Paths stateReferences2; + + if (!isRealisablePath(txn, storePath)) + throw Error(format("path `%1%' is not valid") % storePath); + nixDB.queryStrings(txn, dbStateReferences, storePath, stateReferences2); + stateReferences.insert(stateReferences2.begin(), stateReferences2.end()); +} + +void LocalStore::queryStateReferences(const Path & storePath, + PathSet & stateReferences) +{ + nix::queryStateReferences(noTxn, storePath, stateReferences); +} + + +//TODO getStateReferrers.... void queryReferrers(const Transaction & txn, const Path & storePath, PathSet & referrers) @@ -466,7 +491,7 @@ bool isStateComponent(const Path & storePath) string deriver; string data; - return nixDB.queryString(noTxn, dbStateInfo, storePath, data); //update the dbinfo db. (maybe TODO) + return nixDB.queryString(noTxn, dbStateInfo, storePath, data); } bool LocalStore::isStateComponent(const Path & storePath) @@ -478,7 +503,7 @@ bool LocalStore::isStateComponent(const Path & storePath) bool isStateDrv(const Path & drvPath) { Derivation drv = derivationFromPath(drvPath); //maybe redirect the out path to isStateComponent? - if (drv.outputs.size() != 0) + if (drv.stateOutputs.size() != 0) return true; else return false; @@ -697,6 +722,7 @@ void registerValidPath(const Transaction & txn, info.path = path; info.hash = hash; info.references = references; + info.stateReferences = stateReferences; info.deriver = deriver; ValidPathInfos infos; infos.push_back(info); @@ -720,7 +746,7 @@ void registerValidPaths(const Transaction & txn, debug(format("registering path `%1%'") % i->path); setHash(txn, i->path, i->hash); - setReferences(txn, i->path, i->references); + setReferences(txn, i->path, i->references, i->stateReferences); /* Check that all referenced paths are also valid (or about to become valid). */ @@ -735,7 +761,7 @@ void registerValidPaths(const Transaction & txn, } -/* Invalidate a path. The caller is responsible for checking that +/* Invalidate a path. The caller is responsible for checking that there are no referrers. */ static void invalidatePath(Transaction & txn, const Path & path) { @@ -746,11 +772,11 @@ static void invalidatePath(Transaction & txn, const Path & path) if there are no substitutes for this path. This maintains the cleanup invariant. */ if (querySubstitutes(txn, path).size() == 0) { - setReferences(txn, path, PathSet()); - nixDB.delPair(txn, dbDerivers, path); + setReferences(txn, path, PathSet(), PathSet()); + nixDB.delPair(txn, dbDerivers, path); //TODO!!!!! also for state Derivers!!!!!!!!!!!!!!! } - nixDB.delPair(txn, dbValidPaths, path); + nixDB.delPair(txn, dbValidPaths, path); //Always? } @@ -1152,7 +1178,7 @@ void verifyStore(bool checkContents) if (realisablePaths.find(*i) == realisablePaths.end()) { printMsg(lvlError, format("removing references entry for unrealisable path `%1%'") % *i); - setReferences(txn, *i, PathSet()); + setReferences(txn, *i, PathSet(), PathSet()); } else { bool isValid = validPaths.find(*i) != validPaths.end(); @@ -1206,11 +1232,15 @@ void verifyStore(bool checkContents) else { PathSet references; queryReferences(txn, from, references); + + PathSet stateReferences; + //TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + if (find(references.begin(), references.end(), to) == references.end()) { printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'") % from % to); references.insert(to); - setReferences(txn, from, references); + setReferences(txn, from, references, stateReferences); } } @@ -1285,59 +1315,6 @@ vector LocalStore::getStatePathsInterval(const PathSet & statePaths) } -//TODO direct or all recursive parameter -//TODO check if these are state components -//TODO CHECK FOR DERIVATION INSTEAD OF -PathSet getStateReferencesClosure(const Path & drvpath) -{ - PathSet empty; - - //get all ... - - return empty; // getStateReferencesClosure_(drvpath, empty); -} - -PathSet getStateReferencesClosure_(const Path & drvpath, PathSet & paths) -{ - Transaction txn(nixDB); //TODO should u do a transaction here? ... this might delay the process ... - - //Derivation drv = derivationFromPath(derivationPath); - - //for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) - - /* - for (Strings::iterator i = data.begin(); i != data.end(); ++i) - { - string storePath = *i; - - bool alreadyExists = false; - for (PathSet::iterator j = paths.begin(); j != paths.end(); ++j) //Get also the references of this dep. - { - string checkPath = *j; - if(storePath == checkPath){ - alreadyExists = true; - break; - } - } - - if( !alreadyExists ){ - printMsg(lvlError, format("References: `%1%'") % storePath); - paths.insert(storePath); - PathSet rec = getStateReferencesClosure_(storePath, paths); //go recursive - //paths = mergePathSets(paths, rec); //merge - } - } - */ - - txn.commit(); - return paths; -} - -PathSet LocalStore::getStateReferencesClosure(const Path & path) -{ - return nix::getStateReferencesClosure(path); -} - //Merges a new .... into the list TODO //This is all about one store path @@ -1360,7 +1337,7 @@ PathSet mergeNewDerivationIntoList(const Path & storepath, const Path & newdrv, if(identifier == getIdentifier && getUser == user) //only insert if it doenst already exist { if(deleteDrvs){ - printMsg(lvlError, format("Deleting decrepated state derivation: %1% with identifier %2% and user %3%") % drv % identifier % user); + printMsg(lvlTalkative, format("Deleting decrepated state derivation: %1% with identifier %2% and user %3%") % drv % identifier % user); deletePath(drv); //Deletes the DRV from DISK! } } @@ -1372,6 +1349,27 @@ PathSet mergeNewDerivationIntoList(const Path & storepath, const Path & newdrv, return newdrvs; } +/* We register in this database which _component_ paths are state paths + * This does not mean these paths are already valid!, you should look in derivivers for that + */ +void registerMaybeStatePath(const Path & drvPath) +{ + if(!isStateDrv(drvPath)) + return; + + printMsg(lvlError, format("derivation aaa: %1% ") % drvPath); + + + Derivation drv = derivationFromPath(drvPath); + Path storePath = drv.outputs["out"].path; + + nixDB.setString(noTxn, dbStateInfo, storePath, ""); //update the dbinfo db. (maybe TODO) +} + +void LocalStore::registerMaybeStatePath(const Path & drvPath) +{ + nix::registerMaybeStatePath(drvPath); +} /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */ @@ -1448,7 +1446,7 @@ static void upgradeStore07() printMsg(lvlError, format("warning: conflicting references for `%1%'") % path); if (references != prevReferences) - setReferences(txn, path, references); + setReferences(txn, path, references, PathSet()); } std::cerr << "."; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f72da0b3b..17c50f2ab 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -46,7 +46,9 @@ public: Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); - + + void queryStateReferences(const Path & storePath, PathSet & stateReferences); + void queryReferrers(const Path & path, PathSet & referrers); Path addToStore(const Path & srcPath, bool fixed = false, @@ -80,7 +82,7 @@ public: vector getStatePathsInterval(const PathSet & statePaths); - PathSet getStateReferencesClosure(const Path & path); + void registerMaybeStatePath(const Path & drvPath); bool isStateComponent(const Path & path); @@ -119,6 +121,7 @@ struct ValidPathInfo Path deriver; Hash hash; PathSet references; + PathSet stateReferences; }; typedef list ValidPathInfos; @@ -142,7 +145,7 @@ bool isValidPathTxn(const Transaction & txn, const Path & path); /* Sets the set of outgoing FS references for a store path. Use with care! */ void setReferences(const Transaction & txn, const Path & path, - const PathSet & references); + const PathSet & references, const PathSet & stateReferences); /* Sets the deriver of a store path. Use with care! */ void setDeriver(const Transaction & txn, const Path & path, @@ -185,8 +188,6 @@ void addStateDeriver(const Transaction & txn, const Path & storePath, const Path /* TODO */ PathSet mergeNewDerivationIntoList(const Path & storepath, const Path & newdrv, const PathSet drvs, bool deleteDrvs = false); -/* TODO */ -//PathSet getStateReferencesClosure_(const Path & drvpath, PathSet & paths); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6bb2f0e88..cd3e955e7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -200,6 +200,19 @@ void RemoteStore::queryReferences(const Path & path, references.insert(references2.begin(), references2.end()); } +void RemoteStore::queryStateReferences(const Path & path, + PathSet & stateReferences) +{ + writeInt(wopQueryStateReferences, to); + writeString(path, to); + processStderr(); + + //TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +// PathSet references2 = readStorePaths(from); +// references.insert(references2.begin(), references2.end()); +} + + void RemoteStore::queryReferrers(const Path & path, PathSet & referrers) @@ -390,10 +403,9 @@ vector RemoteStore::getStatePathsInterval(const PathSet & statePaths) } //TODO -PathSet RemoteStore::getStateReferencesClosure(const Path & path) +void RemoteStore::registerMaybeStatePath(const Path & drvPath) { - PathSet empty; - return empty; + } bool RemoteStore::isStateComponent(const Path & path) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9a1c54f31..a8dbe8b77 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -35,6 +35,8 @@ public: void queryReferences(const Path & path, PathSet & references); + void queryStateReferences(const Path & storePath, PathSet & stateReferences); + void queryReferrers(const Path & path, PathSet & referrers); Path addToStore(const Path & srcPath, bool fixed = false, @@ -68,7 +70,7 @@ public: vector getStatePathsInterval(const PathSet & statePaths); - PathSet getStateReferencesClosure(const Path & path); + void registerMaybeStatePath(const Path & drvPath); bool isStateComponent(const Path & path); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b36901e4a..82ba2212a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -73,6 +73,10 @@ public: The result is not cleared. */ virtual void queryReferences(const Path & path, PathSet & references) = 0; + + /* Queries the set of outgoing FS state-references for a store path. + The result is not cleared. */ + virtual void queryStateReferences(const Path & storePath, PathSet & stateReferences) = 0; /* Queries the set of incoming FS references for a store path. The result is not cleared. */ @@ -189,7 +193,7 @@ public: virtual vector getStatePathsInterval(const PathSet & statePaths) = 0; /* TODO */ - virtual PathSet getStateReferencesClosure(const Path & path) = 0; + virtual void registerMaybeStatePath(const Path & drvPath) = 0; /* TODO */ virtual bool isStateComponent(const Path & path) = 0; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 0126c8d59..2d064cca9 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -16,7 +16,9 @@ typedef enum { wopHasSubstitutes, wopQueryPathHash, wopQueryReferences, + wopQueryStateReferences, wopQueryReferrers, + wopQueryStateReferrers, wopAddToStore, wopAddTextToStore, wopBuildDerivations, diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index 9fe9a3103..7b09e2b5a 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -22,7 +22,7 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs); /************************* Build time Functions ******************************/ - +//a /************************* Build time Functions ******************************/ @@ -175,10 +175,8 @@ static void opRunComponent(Strings opFlags, Strings opArgs) //******************* Afterwards, call the commit script (recursively) //get dependecies (if neccecary | recusively) of all state components that need to be updated - PathSet paths = store->getStateReferencesClosure(derivationPath); - - - //TODO nix-store -q --tree $(nix-store -qd /nix/store/6x6glnb9idn53yxfqrz6wq53459vv3qd-firefox-2.0.0.3/) + // + //TODO nix-store -qR $(nix-store -qd /nix/store/6x6glnb9idn53yxfqrz6wq53459vv3qd-firefox-2.0.0.3/) //Transaction txn; @@ -293,6 +291,7 @@ static void opRunComponent(Strings opFlags, Strings opArgs) "commit-script"); } +/* PathSet getStateReferencesClosure_(const Path & drvpath, PathSet & drvPaths, const PathSet & paths) { Derivation drv = derivationFromPath(drvpath); @@ -323,7 +322,7 @@ PathSet getStateReferencesClosure(const Path & drvpath) return drvRefs; } - + */ diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index df84a2a76..7355835d0 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -32,6 +32,7 @@ Query flags: --outputs: query the output paths of a Nix derivation (default) --requisites / -R: print all paths necessary to realise a path --references: print all paths referenced by the given path + --references-state: print all state paths referenced by the given path --referrers: print all paths directly refering to the given path --referrers-closure: print all paths (in)directly refering to the given path --tree: print a tree showing the dependency graph of the given paths diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c7cad18d6..d41daf0df 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -257,7 +257,7 @@ static void printTree(const Path & path, /* Perform various sorts of queries. */ static void opQuery(Strings opFlags, Strings opArgs) { - enum { qOutputs, qRequisites, qReferences, qReferrers + enum { qOutputs, qRequisites, qReferences, qStateReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash , qTree, qGraph, qResolve } query = qOutputs; bool useOutput = false; @@ -270,6 +270,7 @@ static void opQuery(Strings opFlags, Strings opArgs) if (*i == "--outputs") query = qOutputs; else if (*i == "--requisites" || *i == "-R") query = qRequisites; else if (*i == "--references") query = qReferences; + else if (*i == "--references-state") query = qStateReferences; else if (*i == "--referrers" || *i == "--referers") query = qReferrers; else if (*i == "--referrers-closure" || *i == "--referers-closure") query = qReferrersClosure; else if (*i == "--deriver" || *i == "-d") query = qDeriver; @@ -305,6 +306,7 @@ static void opQuery(Strings opFlags, Strings opArgs) case qRequisites: case qReferences: + case qStateReferences: case qReferrers: case qReferrersClosure: { PathSet paths; @@ -315,6 +317,7 @@ static void opQuery(Strings opFlags, Strings opArgs) if (query == qRequisites) storePathRequisites(path, includeOutputs, paths); else if (query == qReferences) store->queryReferences(path, paths); + else if (query == qStateReferences) store->queryStateReferences(path, paths); else if (query == qReferrers) store->queryReferrers(path, paths); else if (query == qReferrersClosure) computeFSClosure(path, paths, true); } @@ -424,6 +427,10 @@ static void opRegisterSubstitutes(Strings opFlags, Strings opArgs) Path srcPath; Substitute sub; PathSet references; + + //TODO TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1 + PathSet stateReferences; + getline(cin, srcPath); if (cin.eof()) break; getline(cin, sub.deriver); @@ -443,7 +450,7 @@ static void opRegisterSubstitutes(Strings opFlags, Strings opArgs) } if (!cin || cin.eof()) throw Error("missing input"); registerSubstitute(txn, srcPath, sub); - setReferences(txn, srcPath, references); + setReferences(txn, srcPath, references, stateReferences); } txn.commit(); diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index bf1a8923b..b390eb4b9 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -276,6 +276,20 @@ static void performOp(Source & from, Sink & to, unsigned int op) writeStringSet(paths, to); break; } + + case wopQueryStateReferences: + case wopQueryStateReferrers: { + Path path = readStorePath(from); + startWork(); + PathSet paths; + if (op == wopQueryStateReferences) + store->queryStateReferences(path, paths); + //else + // store->queryStateReferrers(path, paths); //TODO TODO implemnt function, and then commen out !!!!!!!!!!!!!!!!!!!!! + stopWork(); + writeStringSet(paths, to); + break; + } case wopAddToStore: { /* !!! uberquick hack */