Bugfix: Before adjusting registerValidPath to also be able to take state paths

src/libstore/build.cc

@@ -1759,9 +1759,10 @@ void DerivationGoal::computeClosure()
      * [scan for and state references and component references in the state path]	//3,4
      */
     
-    //TODO we scan for each output, be then we do multiple scans inside for the state path .....
+    //TODO we scan for each output-path, be then we do multiple scans on the state path if there are more ouputs paths then 1.....
     
-    //TODO !!!!!!!!! we don not ONLY need to scan all outputs, but also (recursively) the component references in the state folders 
+    //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    // WE NOW ONLY REGISTER COMPONETS PATHS WITH registerValidPath BUT WE SHOULD ALSO REGISTER STATE PAHTS AS VALID AND SET THEIR REFERENCES !!!!!!!!!!!!!!!!!!!!!!!
     
     for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)
     {    
@@ -1769,7 +1770,7 @@ void DerivationGoal::computeClosure()
     	
     	/* For this state-output path, find the references to other paths contained in it. 
 		 * Get the state paths (instead of out paths) from all components, and then call
-		 * scanForStateReferences().
+		 * scanForReferences().
 		 */
 		PathSet allStatePaths;
 		for (PathSet::const_iterator i = allPaths.begin(); i != allPaths.end(); i++){
@@ -1783,7 +1784,7 @@ void DerivationGoal::computeClosure()
 		}
 		
 		//We scan for state references in the component path
-		PathSet all_state_references = scanForStateReferences(path, allStatePaths);
+		PathSet all_state_references = scanForReferences(path, allStatePaths);
 			
 		//If state is enabled: Seaches for state and component references in the state path 				 
 		if(isStateDrvTxn(txn, drv)){
@@ -1791,7 +1792,7 @@ void DerivationGoal::computeClosure()
 			printMsg(lvlTalkative, format("scanning for component and state references inside `%1%'") % statePath);
 			
 			PathSet state_references = scanForReferences(statePath, allPaths);
-			PathSet state_stateReferences = scanForStateReferences(statePath, allStatePaths);
+			PathSet state_stateReferences = scanForReferences(statePath, allStatePaths);
 			all_state_references = mergePathSets(all_state_references, mergePathSets(state_references, state_stateReferences));
 			
 			statePaths[path] = statePath; 
@@ -1799,15 +1800,13 @@ void DerivationGoal::computeClosure()
 		else
 			statePaths[path] = "";
 
-		for (PathSet::const_iterator i = allStatePaths.begin(); i != allStatePaths.end(); i++){
+		//debugging
+		for (PathSet::const_iterator i = allStatePaths.begin(); i != allStatePaths.end(); i++)
 			debug(format("all possible StatePaths: %1%") % (*i));
-		}
+		for (PathSet::const_iterator i = all_state_references.begin(); i != all_state_references.end(); i++)
-		for (PathSet::const_iterator i = all_state_references.begin(); i != all_state_references.end(); i++){
 			debug(format("state References scanned: %1%") % (*i));
-		}
 		
 		allStateReferences[path] = all_state_references;
-		
     }
     
     for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)

src/libstore/local-store.cc

@@ -1,3 +1,4 @@
+#include "references.hh"
 #include "config.h"
 #include "local-store.hh"
 #include "util.hh"
@@ -96,7 +97,6 @@ static TableId dbSubstitutes = 0;
    derivations specifying an expected hash). */
 static TableId dbDerivers = 0;
 
-
 /* dbStateCounters :: StatePath -> Int
 
    This table lists the state folders that state managed components
@@ -104,8 +104,7 @@ static TableId dbDerivers = 0;
 */
 static TableId dbStateCounters = 0;
 
-
+/* dbStateInfo :: Path -> DerivationPath
-/* dbStateCounters :: Path -> String
 
    This table lists the all the state managed components, TODO we could store the entire DRV in here in the future
     
@@ -852,10 +851,10 @@ void registerValidPaths(const Transaction & txn, const ValidPathInfos & infos)
         assertStorePath(i->path);
 
         debug(format("registering path `%1%'") % i->path);
-        setHash(txn, i->path, i->hash);
+        setHash(txn, i->path, i->hash);								//set path valid
 
 		if (i->statePath != "")
-			setStateValid(txn, i->statePath, i->deriver);			//if the key exists, we know that the state path is valid, we set the value to the drvPath 
+			setStateValid(txn, i->statePath, i->deriver);			//set state path valid 
 
         setReferences(txn, i->path, i->references, i->stateReferences);
         
@@ -1504,6 +1503,11 @@ void storePathRequisites(const Path & storePath, const bool includeOutputs, Path
     }
 }
 
+void LocalStore::storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState)
+{
+    return nix::storePathRequisites(storePath, includeOutputs, paths, withState);
+}
+
 /*
  * Same as storePathRequisites with withState=true, but now only returns the state paths
  */
@@ -1523,17 +1527,15 @@ void LocalStore::storePathStateRequisitesOnly(const Path & storePath, const bool
 {
 	nix::storePathStateRequisitesOnlyTxn(noTxn, storePath, includeOutputs, statePaths);
 }
-void LocalStore::storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState)
-{
-    return nix::storePathRequisites(storePath, includeOutputs, paths, withState);
-}
 
-void convertStatePathsToDerivations(const Transaction & txn, const Path & storePath)
-{
-	//TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-}
 
+/*
+ * TODO
+ * 
+ * Not used yet ......................... (should we use it .... ???)
+ * 
+ */
 void getDependenciesAtBuildTime(const Transaction & txn, const Path & drvPath)
 {
  	Derivation drv = derivationFromPath(drvPath);	
@@ -1577,6 +1579,64 @@ void getDependenciesAtBuildTime(const Transaction & txn, const Path & drvPath)
     	printMsg(lvlError, format("ALLPATHS2: %1%") % *i);
 }
 
+void scanForAllReferences(const Transaction & txn, const Path & statePath)
+{
+	//get all possible state and component references
+	
+	Paths referencesKeys;
+	Paths referencesKeys2;
+	Paths stateReferencesKeys;
+    nixDB.enumTable(txn, dbReferences, referencesKeys);
+    nixDB.enumTable(txn, dbStateReferences, stateReferencesKeys);
+    
+    for (Paths::iterator i = stateReferencesKeys.begin(); i != stateReferencesKeys.end(); ++i)
+    	printMsg(lvlError, format("STATE: %1%") % *i);
+    
+    //Remove derivation paths .....
+    for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i){
+    	string path = *i;
+    	//printMsg(lvlError, format("refkey: %1%") % path);
+    	if(path.substr(path.length() - 4,path.length()) != ".drv")		//TODO HACK: we should have a typed table or a seperate table ....
+    		referencesKeys2.push_back(path);
+    }
+
+	//Merge
+	PathSet scanPaths =	mergePathSets(PathSet(referencesKeys2.begin(), referencesKeys2.end()), 
+									  PathSet(stateReferencesKeys.begin(), stateReferencesKeys.end()));  
+    
+    //for (PathSet::iterator i = scanPaths.begin(); i != scanPaths.end(); ++i)
+    //	printMsg(lvlError, format("SCANNED: %1%") % *i);
+    
+    //Scan in statePath
+    PathSet scannedReferences = scanForReferences(statePath, scanPaths);
+    
+    for (PathSet::iterator i = scannedReferences.begin(); i != scannedReferences.end(); ++i)
+    	printMsg(lvlError, format("RESULT: %1%") % *i);
+}
+
+void LocalStore::scanForAllReferences(const Path & statePath)
+{
+    return nix::scanForAllReferences(noTxn, statePath);
+}
+
+void scanForAllReferencesRecusively(const Transaction & txn, const Path & storePath)
+{
+	//get all state references
+	
+	//call scanForAllReferences on all
+	
+	//compare results with the current registered component and state paths
+	
+	//update the extra references in a new table??? why??? 
+	//(remember we need to keep the old as the basis, and things can change, the db is not consisten anymore then ....) and error if neseccary
+}
+
+void LocalStore::scanForAllReferencesRecusively(const Path & storePath)
+{
+    return nix::scanForAllReferencesRecusively(noTxn, storePath);
+}
+
+
 /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */
 static void upgradeStore07()
 {

src/libstore/local-store.hh

@@ -100,6 +100,10 @@ public:
 	
 	void storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths);
 	
+	void scanForAllReferences(const Path & statePath);
+	
+	void scanForAllReferencesRecusively(const Path & storePath);
+	
 };
 
 
@@ -210,8 +214,6 @@ bool isStateDrvPathTxn(const Transaction & txn, const Path & drvPath);
 
 bool isStateDrvTxn(const Transaction & txn, const Derivation & drv);
  
-void convertStatePathsToDerivations(const Transaction & txn, const Path & storePath);
- 
 }
 
 

src/libstore/references.cc

@@ -152,17 +152,5 @@ PathSet scanForReferences(const string & path, const PathSet & paths)
     return found;
 }
 
-/* A wrapper for now, but we may extend */
-PathSet scanForStateReferences(const string & path, const PathSet & statePaths)
-{
-    return scanForReferences(path, statePaths);
-}
-
-PathSet scanForALLReferences(const string & path)
-{
-	PathSet p;
-	throw Error("TODO");
-	return p; 
-}
  
 }

src/libstore/references.hh

@@ -7,10 +7,6 @@ namespace nix {
 
 PathSet scanForReferences(const Path & path, const PathSet & refs);
 
-PathSet scanForStateReferences(const string & path, const PathSet & statePaths);
-
-PathSet scanForALLReferences(const string & path);
-    
 }
 
 #endif /* !__REFERENCES_H */

src/libstore/remote-store.cc

@@ -463,4 +463,18 @@ void RemoteStore::storePathStateRequisitesOnly(const Path & storePath, const boo
 	
 }
 
+//TODO
+void RemoteStore::scanForAllReferences(const Path & statePath)
+{
+	
+}
+
+//TODO	
+void RemoteStore::scanForAllReferencesRecusively(const Path & storePath)
+{
+	
+}
+
+
+
 }

src/libstore/remote-store.hh

@@ -88,6 +88,10 @@ public:
 
 	void storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths);
 	
+	void scanForAllReferences(const Path & statePath);
+	
+	void scanForAllReferencesRecusively(const Path & storePath);
+    
 private:
     AutoCloseFD fdSocket;
     FdSink to;

src/libstore/store-api.hh

@@ -214,10 +214,17 @@ public:
 	/* TODO */
 	virtual bool isStateDrv(const Derivation & drv) = 0;
 	
+	/* TODO */
 	virtual void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState) = 0;
 
+	/* TODO */
 	virtual void storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths) = 0;	
 	
+	/* TODO */
+	virtual void scanForAllReferences(const Path & statePath) = 0;
+	
+	/* TODO */
+	virtual void scanForAllReferencesRecusively(const Path & storePath) = 0;
 };
 
 

src/libutil/util.cc

@@ -1052,41 +1052,20 @@ string trim(const string & s) {
 }
 
 
-//executes a shell command and captures and prints the output.
+//executes a shell command, captures and prints the output.
-
-//TODO , check if we can replace!!! with runProgram  like this: string a = runProgram("whoami", true, s);
-
-/*
- 
- Strings args;
- args.push_back("rsautl");
- args.push_back("-sign");
- args.push_back("-inkey");
- args.push_back(secretKey);
- args.push_back("-in");
- args.push_back(hashFile);
- string signature = runProgram(OPENSSL_PATH, true, args);
-  
- */
-
 void runProgram_AndPrintOutput(Path program, bool searchPath, const Strings & args, const string outputPrefix)
 {
 	string program_output = runProgram(program, searchPath, args);
 
 	//Add the prefix on every line
-	//TODO
-	
 	Strings lines = tokenizeString(program_output, "\n");
 	for (Strings::const_iterator i = lines.begin(); i != lines.end(); ++i){
 		if(trim(*i) != "")
-			printMsg(lvlError, format("[%2%]- %1%") % *i % outputPrefix);
+			printMsg(lvlError, format("[%2%]: %1%") % *i % outputPrefix);
 	}
-	
-	//Remove the trailing \n
-    //size_t found = program_output.find_last_of("\n");
-	//printMsg(lvlError, format("%1%") % program_output.substr(0,found));
 }
 
+//executes a direct shell command (faster)
 void executeShellCommand(const string & command)
 {
 	int kidstatus, deadpid;

src/nix-state/nix-state.cc

@@ -63,7 +63,7 @@ Derivation getDerivation_andCheckArgs_(Strings opFlags, Strings opArgs, Path & c
     isStatePath = store->isStateComponent(componentPath);
       
     //Extract the program arguments
-    string allargs;
+    string allArgs;
     if(opArgs.size() > 1){
 		opArgs.pop_front();
 		allArgs = opArgs.front();
@@ -72,7 +72,7 @@ Derivation getDerivation_andCheckArgs_(Strings opFlags, Strings opArgs, Path & c
 		//Strings progam_args_strings = tokenizeString(allArgs, " ");
     }
 
-	printMsg(lvlError, format("'%1%' - '%2%' - '%3%' - '%4%' - '%5%'") % componentPath % stateIdentifier % binary % username % allargs);
+	printMsg(lvlError, format("'%1%' - '%2%' - '%3%' - '%4%' - '%5%'") % componentPath % stateIdentifier % binary % username % allArgs);
     
     if(isStatePath)
     	derivers = queryDerivers(noTxn, componentPath, stateIdentifier, username);
@@ -200,14 +200,18 @@ static void opRunComponent(Strings opFlags, Strings opArgs)
 	string svnadminbin = nixSVNPath + "/svnadmin";
     
         
-    //Check for locks ... ?
+    //Check for locks ... ? or put locks on the neseccary state components
+    //WARNING: we need to watch out for deadlocks!
 	//add locks ... ?
 	//svn lock ... ?
 
     //get dependecies (if neccecary | recusively) of all state components that need to be updated
-    //TODO maybe also scan the parameters for state or component hashes?
     PathSet drvs = getAllStateDerivationsRecursively(componentPath);
     
+    
+    //TODO maybe also scan the parameters for state or component hashes?
+    //program_args
+	
 	//????	
 	//Transaction txn;
    	//createStoreTransaction(txn);
@@ -215,7 +219,7 @@ static void opRunComponent(Strings opFlags, Strings opArgs)
 	
 	//******************* Run ****************************
 	
-	executeShellCommand(componentPath + binary);	//more efficient way needed ???
+	executeShellCommand(componentPath + binary + " " + program_args);	//more efficient way needed ???
   		
 	//******************* With everything in place, we call the commit script on all statePaths **********************
 	
@@ -366,17 +370,21 @@ void run(Strings args)
 	printMsg(lvlError, format("2: %1%") % bool2string( store->isStateComponent("/nix/store/05441jm8xmsidqm43ivk0micckf0mr2m-nvidiaDrivers") ) );
 	printMsg(lvlError, format("3: %1%") % bool2string( store->isStateDrvPath("/nix/store/2hpx60ibdfv2pslg4rjvp177frijamvi-hellostateworld-1.0.drv") ) );
 	
-	store = openStore();
-	convertStatePathsToDerivations(noTxn, "");
-	return;
-	
 	store = openStore();
 	Path p = store->queryStatePathDrv("/nix/state/6g6kfgimz8szznlshf13s29fn01zp99d-hellohardcodedstateworld-1.0-test2");
 	printMsg(lvlError, format("Result: %1%") % p);
 	return;
 
+	
 	*/
 	
+	string path = "afddsafsdafsdaf.drv";
+	printMsg(lvlError, format("Result: %1%") % path.substr(path.length() - 4,path.length()));
+	
+	store = openStore();
+	store->scanForAllReferences("/nix/state/0qhlpz1ji4gvg3j6nk5vkcddmi3m5x1r-hellohardcodedstateworld-1.0-test2");
+	return;
+	
 	/* test */
 	
     for (Strings::iterator i = args.begin(); i != args.end(); ) {
@@ -399,7 +407,7 @@ void run(Strings args)
 
 		--run-without-commit
 		
-		--backup
+		--backup ?
 		
 		--exclude-commit-paths
 		
@@ -407,7 +415,9 @@ void run(Strings args)
 		
 		--revert-to-state	(recursive revert...)
 		
-		--delete state?
+		--delete-state
+		
+		--share-from
 		
         */