From 04dd3fdf34d3f63c3a86ab380c089f7ef725e9d1 Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Thu, 28 Jun 2007 17:12:02 +0000 Subject: [PATCH] Bugfix: Before adjusting registerValidPath to also be able to take state paths --- src/libstore/build.cc | 19 ++++---- src/libstore/local-store.cc | 88 ++++++++++++++++++++++++++++++------ src/libstore/local-store.hh | 6 ++- src/libstore/references.cc | 12 ----- src/libstore/references.hh | 4 -- src/libstore/remote-store.cc | 14 ++++++ src/libstore/remote-store.hh | 4 ++ src/libstore/store-api.hh | 9 +++- src/libutil/util.cc | 27 ++--------- src/nix-state/nix-state.cc | 32 ++++++++----- 10 files changed, 137 insertions(+), 78 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 573e81bf4..9d724cf9b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1759,9 +1759,10 @@ void DerivationGoal::computeClosure() * [scan for and state references and component references in the state path] //3,4 */ - //TODO we scan for each output, be then we do multiple scans inside for the state path ..... + //TODO we scan for each output-path, be then we do multiple scans on the state path if there are more ouputs paths then 1..... - //TODO !!!!!!!!! we don not ONLY need to scan all outputs, but also (recursively) the component references in the state folders + //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + // WE NOW ONLY REGISTER COMPONETS PATHS WITH registerValidPath BUT WE SHOULD ALSO REGISTER STATE PAHTS AS VALID AND SET THEIR REFERENCES !!!!!!!!!!!!!!!!!!!!!!! for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) { @@ -1769,7 +1770,7 @@ void DerivationGoal::computeClosure() /* For this state-output path, find the references to other paths contained in it. * Get the state paths (instead of out paths) from all components, and then call - * scanForStateReferences(). + * scanForReferences(). */ PathSet allStatePaths; for (PathSet::const_iterator i = allPaths.begin(); i != allPaths.end(); i++){ @@ -1783,7 +1784,7 @@ void DerivationGoal::computeClosure() } //We scan for state references in the component path - PathSet all_state_references = scanForStateReferences(path, allStatePaths); + PathSet all_state_references = scanForReferences(path, allStatePaths); //If state is enabled: Seaches for state and component references in the state path if(isStateDrvTxn(txn, drv)){ @@ -1791,7 +1792,7 @@ void DerivationGoal::computeClosure() printMsg(lvlTalkative, format("scanning for component and state references inside `%1%'") % statePath); PathSet state_references = scanForReferences(statePath, allPaths); - PathSet state_stateReferences = scanForStateReferences(statePath, allStatePaths); + PathSet state_stateReferences = scanForReferences(statePath, allStatePaths); all_state_references = mergePathSets(all_state_references, mergePathSets(state_references, state_stateReferences)); statePaths[path] = statePath; @@ -1799,15 +1800,13 @@ void DerivationGoal::computeClosure() else statePaths[path] = ""; - for (PathSet::const_iterator i = allStatePaths.begin(); i != allStatePaths.end(); i++){ + //debugging + for (PathSet::const_iterator i = allStatePaths.begin(); i != allStatePaths.end(); i++) debug(format("all possible StatePaths: %1%") % (*i)); - } - for (PathSet::const_iterator i = all_state_references.begin(); i != all_state_references.end(); i++){ + for (PathSet::const_iterator i = all_state_references.begin(); i != all_state_references.end(); i++) debug(format("state References scanned: %1%") % (*i)); - } allStateReferences[path] = all_state_references; - } for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index df1e3781b..eb5927d73 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,3 +1,4 @@ +#include "references.hh" #include "config.h" #include "local-store.hh" #include "util.hh" @@ -96,7 +97,6 @@ static TableId dbSubstitutes = 0; derivations specifying an expected hash). */ static TableId dbDerivers = 0; - /* dbStateCounters :: StatePath -> Int This table lists the state folders that state managed components @@ -104,8 +104,7 @@ static TableId dbDerivers = 0; */ static TableId dbStateCounters = 0; - -/* dbStateCounters :: Path -> String +/* dbStateInfo :: Path -> DerivationPath This table lists the all the state managed components, TODO we could store the entire DRV in here in the future @@ -852,10 +851,10 @@ void registerValidPaths(const Transaction & txn, const ValidPathInfos & infos) assertStorePath(i->path); debug(format("registering path `%1%'") % i->path); - setHash(txn, i->path, i->hash); + setHash(txn, i->path, i->hash); //set path valid if (i->statePath != "") - setStateValid(txn, i->statePath, i->deriver); //if the key exists, we know that the state path is valid, we set the value to the drvPath + setStateValid(txn, i->statePath, i->deriver); //set state path valid setReferences(txn, i->path, i->references, i->stateReferences); @@ -1504,6 +1503,11 @@ void storePathRequisites(const Path & storePath, const bool includeOutputs, Path } } +void LocalStore::storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState) +{ + return nix::storePathRequisites(storePath, includeOutputs, paths, withState); +} + /* * Same as storePathRequisites with withState=true, but now only returns the state paths */ @@ -1523,17 +1527,15 @@ void LocalStore::storePathStateRequisitesOnly(const Path & storePath, const bool { nix::storePathStateRequisitesOnlyTxn(noTxn, storePath, includeOutputs, statePaths); } -void LocalStore::storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState) -{ - return nix::storePathRequisites(storePath, includeOutputs, paths, withState); -} -void convertStatePathsToDerivations(const Transaction & txn, const Path & storePath) -{ - //TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -} + +/* + * TODO + * + * Not used yet ......................... (should we use it .... ???) + * + */ void getDependenciesAtBuildTime(const Transaction & txn, const Path & drvPath) { Derivation drv = derivationFromPath(drvPath); @@ -1577,6 +1579,64 @@ void getDependenciesAtBuildTime(const Transaction & txn, const Path & drvPath) printMsg(lvlError, format("ALLPATHS2: %1%") % *i); } +void scanForAllReferences(const Transaction & txn, const Path & statePath) +{ + //get all possible state and component references + + Paths referencesKeys; + Paths referencesKeys2; + Paths stateReferencesKeys; + nixDB.enumTable(txn, dbReferences, referencesKeys); + nixDB.enumTable(txn, dbStateReferences, stateReferencesKeys); + + for (Paths::iterator i = stateReferencesKeys.begin(); i != stateReferencesKeys.end(); ++i) + printMsg(lvlError, format("STATE: %1%") % *i); + + //Remove derivation paths ..... + for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i){ + string path = *i; + //printMsg(lvlError, format("refkey: %1%") % path); + if(path.substr(path.length() - 4,path.length()) != ".drv") //TODO HACK: we should have a typed table or a seperate table .... + referencesKeys2.push_back(path); + } + + //Merge + PathSet scanPaths = mergePathSets(PathSet(referencesKeys2.begin(), referencesKeys2.end()), + PathSet(stateReferencesKeys.begin(), stateReferencesKeys.end())); + + //for (PathSet::iterator i = scanPaths.begin(); i != scanPaths.end(); ++i) + // printMsg(lvlError, format("SCANNED: %1%") % *i); + + //Scan in statePath + PathSet scannedReferences = scanForReferences(statePath, scanPaths); + + for (PathSet::iterator i = scannedReferences.begin(); i != scannedReferences.end(); ++i) + printMsg(lvlError, format("RESULT: %1%") % *i); +} + +void LocalStore::scanForAllReferences(const Path & statePath) +{ + return nix::scanForAllReferences(noTxn, statePath); +} + +void scanForAllReferencesRecusively(const Transaction & txn, const Path & storePath) +{ + //get all state references + + //call scanForAllReferences on all + + //compare results with the current registered component and state paths + + //update the extra references in a new table??? why??? + //(remember we need to keep the old as the basis, and things can change, the db is not consisten anymore then ....) and error if neseccary +} + +void LocalStore::scanForAllReferencesRecusively(const Path & storePath) +{ + return nix::scanForAllReferencesRecusively(noTxn, storePath); +} + + /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */ static void upgradeStore07() { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index b1087960e..ec42470f0 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -100,6 +100,10 @@ public: void storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths); + void scanForAllReferences(const Path & statePath); + + void scanForAllReferencesRecusively(const Path & storePath); + }; @@ -209,8 +213,6 @@ bool isStateComponentTxn(const Transaction & txn, const Path & path); bool isStateDrvPathTxn(const Transaction & txn, const Path & drvPath); bool isStateDrvTxn(const Transaction & txn, const Derivation & drv); - -void convertStatePathsToDerivations(const Transaction & txn, const Path & storePath); } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 9da545af1..19cb288bd 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -152,17 +152,5 @@ PathSet scanForReferences(const string & path, const PathSet & paths) return found; } -/* A wrapper for now, but we may extend */ -PathSet scanForStateReferences(const string & path, const PathSet & statePaths) -{ - return scanForReferences(path, statePaths); -} - -PathSet scanForALLReferences(const string & path) -{ - PathSet p; - throw Error("TODO"); - return p; -} } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 71db44688..55da8f7fd 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -7,10 +7,6 @@ namespace nix { PathSet scanForReferences(const Path & path, const PathSet & refs); -PathSet scanForStateReferences(const string & path, const PathSet & statePaths); - -PathSet scanForALLReferences(const string & path); - } #endif /* !__REFERENCES_H */ diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7e2472776..ee9ac52c9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -463,4 +463,18 @@ void RemoteStore::storePathStateRequisitesOnly(const Path & storePath, const boo } +//TODO +void RemoteStore::scanForAllReferences(const Path & statePath) +{ + +} + +//TODO +void RemoteStore::scanForAllReferencesRecusively(const Path & storePath) +{ + +} + + + } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9d1a22f27..7afaefcbf 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -87,6 +87,10 @@ public: void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState); void storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths); + + void scanForAllReferences(const Path & statePath); + + void scanForAllReferencesRecusively(const Path & storePath); private: AutoCloseFD fdSocket; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4f4131540..f24d1ad6e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -214,10 +214,17 @@ public: /* TODO */ virtual bool isStateDrv(const Derivation & drv) = 0; + /* TODO */ virtual void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState) = 0; + /* TODO */ virtual void storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths) = 0; - + + /* TODO */ + virtual void scanForAllReferences(const Path & statePath) = 0; + + /* TODO */ + virtual void scanForAllReferencesRecusively(const Path & storePath) = 0; }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5e5b79b7c..db1fc0fc4 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1052,41 +1052,20 @@ string trim(const string & s) { } -//executes a shell command and captures and prints the output. - -//TODO , check if we can replace!!! with runProgram like this: string a = runProgram("whoami", true, s); - -/* - - Strings args; - args.push_back("rsautl"); - args.push_back("-sign"); - args.push_back("-inkey"); - args.push_back(secretKey); - args.push_back("-in"); - args.push_back(hashFile); - string signature = runProgram(OPENSSL_PATH, true, args); - - */ - +//executes a shell command, captures and prints the output. void runProgram_AndPrintOutput(Path program, bool searchPath, const Strings & args, const string outputPrefix) { string program_output = runProgram(program, searchPath, args); //Add the prefix on every line - //TODO - Strings lines = tokenizeString(program_output, "\n"); for (Strings::const_iterator i = lines.begin(); i != lines.end(); ++i){ if(trim(*i) != "") - printMsg(lvlError, format("[%2%]- %1%") % *i % outputPrefix); + printMsg(lvlError, format("[%2%]: %1%") % *i % outputPrefix); } - - //Remove the trailing \n - //size_t found = program_output.find_last_of("\n"); - //printMsg(lvlError, format("%1%") % program_output.substr(0,found)); } +//executes a direct shell command (faster) void executeShellCommand(const string & command) { int kidstatus, deadpid; diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index 6563f4da5..3f3de6b89 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -63,7 +63,7 @@ Derivation getDerivation_andCheckArgs_(Strings opFlags, Strings opArgs, Path & c isStatePath = store->isStateComponent(componentPath); //Extract the program arguments - string allargs; + string allArgs; if(opArgs.size() > 1){ opArgs.pop_front(); allArgs = opArgs.front(); @@ -72,7 +72,7 @@ Derivation getDerivation_andCheckArgs_(Strings opFlags, Strings opArgs, Path & c //Strings progam_args_strings = tokenizeString(allArgs, " "); } - printMsg(lvlError, format("'%1%' - '%2%' - '%3%' - '%4%' - '%5%'") % componentPath % stateIdentifier % binary % username % allargs); + printMsg(lvlError, format("'%1%' - '%2%' - '%3%' - '%4%' - '%5%'") % componentPath % stateIdentifier % binary % username % allArgs); if(isStatePath) derivers = queryDerivers(noTxn, componentPath, stateIdentifier, username); @@ -200,13 +200,17 @@ static void opRunComponent(Strings opFlags, Strings opArgs) string svnadminbin = nixSVNPath + "/svnadmin"; - //Check for locks ... ? + //Check for locks ... ? or put locks on the neseccary state components + //WARNING: we need to watch out for deadlocks! //add locks ... ? //svn lock ... ? //get dependecies (if neccecary | recusively) of all state components that need to be updated - //TODO maybe also scan the parameters for state or component hashes? PathSet drvs = getAllStateDerivationsRecursively(componentPath); + + + //TODO maybe also scan the parameters for state or component hashes? + //program_args //???? //Transaction txn; @@ -215,7 +219,7 @@ static void opRunComponent(Strings opFlags, Strings opArgs) //******************* Run **************************** - executeShellCommand(componentPath + binary); //more efficient way needed ??? + executeShellCommand(componentPath + binary + " " + program_args); //more efficient way needed ??? //******************* With everything in place, we call the commit script on all statePaths ********************** @@ -366,17 +370,21 @@ void run(Strings args) printMsg(lvlError, format("2: %1%") % bool2string( store->isStateComponent("/nix/store/05441jm8xmsidqm43ivk0micckf0mr2m-nvidiaDrivers") ) ); printMsg(lvlError, format("3: %1%") % bool2string( store->isStateDrvPath("/nix/store/2hpx60ibdfv2pslg4rjvp177frijamvi-hellostateworld-1.0.drv") ) ); - store = openStore(); - convertStatePathsToDerivations(noTxn, ""); - return; - store = openStore(); Path p = store->queryStatePathDrv("/nix/state/6g6kfgimz8szznlshf13s29fn01zp99d-hellohardcodedstateworld-1.0-test2"); printMsg(lvlError, format("Result: %1%") % p); return; + */ + string path = "afddsafsdafsdaf.drv"; + printMsg(lvlError, format("Result: %1%") % path.substr(path.length() - 4,path.length())); + + store = openStore(); + store->scanForAllReferences("/nix/state/0qhlpz1ji4gvg3j6nk5vkcddmi3m5x1r-hellohardcodedstateworld-1.0-test2"); + return; + /* test */ for (Strings::iterator i = args.begin(); i != args.end(); ) { @@ -399,7 +407,7 @@ void run(Strings args) --run-without-commit - --backup + --backup ? --exclude-commit-paths @@ -407,7 +415,9 @@ void run(Strings args) --revert-to-state (recursive revert...) - --delete state? + --delete-state + + --share-from */