Changed commit script: it recursively walkes through all dirs itself now, uses svn stat where needed, and doesnt use svn add *,svn revert anymore and is much faster

scripts/nix-statecommit.sh.in

@@ -1,5 +1,5 @@
 #! /bin/sh
-# we cant do a -e since ...
+# we cant do a -e since ... svn can fail ??? TODO...
 
 #check if there are enough arguments, if not, exit with an error
 
@@ -17,50 +17,59 @@ nonversionedpaths=( $4 )
 checkouts=( $5 )
 
 #echo svnbin: $svnbin
-#echo subversionedpaths: $subversionedpaths
+#echo subversionedpaths: ${subversionedpaths[@]}
-#echo subversionedpathsCommitBools: $subversionedpathsCommitBools
+#echo subversionedpathsCommitBools: ${subversionedpathsCommitBools[@]}
-#echo nonversionedpaths: $nonversionedpaths
+#echo nonversionedpaths: ${nonversionedpaths[@]}
-#echo checkouts: $checkouts
+#echo checkouts: ${checkouts[@]}
 
-
-
-#TODO
 #
-# silence "is already under version control" messages
-# after a "revert $x" you can silence the "skipped '$x'"
 #
-#TODO
+#
+#
+function subversionSingleStateDir {
 
+	  excludelist=( "." ".." ".svn" );
 
+	  cd $1;
+	  echo cd $1;
 	  
-i=0
+	  empty=$(ls)
-for path in ${subversionedpaths[@]}
-do
-   if test -d $path; then														#if the dir doesnt exist, than we dont hav to do anything
-      cd $path;
 	  
-      output=$($svnbin stat 2>&1 | grep "is not a working copy");
+	  if [ "$empty" = "" ] ; then
-      if [ "$output" != "" ] ; then       										#if the dir exists but is not yet an svn dir: create repos, if it doenst exits (is removed or something) than we dont do anything
+	  	allsubitems=();															#no subfiles / dirs
-          $debug ${checkouts[$i]};
+	  else
+	  	allsubitems=( $(echo *) $(echo .*) )									#there are subfiles / dirs,also adds hidden items
 	  fi
 
-      if [ "${subversionedpathsCommitBools[$i]}" = "true" ]; then				#Check if we need to commit this folder
+	  for subitem in ${allsubitems[@]}
+	  do
+	  	  if [ "$subitem" = ".svn" ]; then
+	  	  		allsubitems=( $(svn -N stat | sed -n '/^?/p' | sed 's/?     //' | tr -d "\12") )		#there are subfiles, and theyre already versioned
+	  	  fi
+	  done
 	  
-          echo "Entering $path"
+	  #echo "Allsubitems ${allsubitems[@]}"
 	  																		
-		  allsubitems=( $(echo *) )												#TODO, maybe also add hidden files starting with a '.' , but we dont want to add .svn dirs
-		  																		#TODO2 maybe do something with svn stat to speed up ?
 	  subitems=();
 	  for subitem in ${allsubitems[@]}										#add all, were going to exlucde explicity stated direct versioned-subdirs or explicity stated nonversioned-subdirs
 	     do																	#this is only to prevent some warnings, ultimately we would like svn add to have a option 'exclude dirs'
-			  subitem="$(pwd)/$subitem";
+
+		  exclude=0;
+		  
+	  	  for excl in ${excludelist[@]}										#check if the subitem is in the list of excluded dirs
+		  do
+		  	  if [ "$excl" = "$subitem" ]; then
+		  	    exclude=1;
+		  	    #echo "exclude $excl"
+		  	  fi
+		  done	  	  
+
+		  subitem="$(pwd)/$subitem";										#create the full path
 		  
 		  if test -d $subitem; then											#the subitem (file or a dir) may be a dir, so we add a / to the end
 		  	subitem="$subitem/";
 		  fi
 		  
-			  exclude=0;
-			  
 		  for svnp in ${subversionedpaths[@]}								#check if the subitem is in the list of subverioned paths
 		  do
 		  	  if [ "$svnp" = "$subitem" ]; then
@@ -82,23 +91,44 @@ do
          fi
       done  
 	  
-		  if [ "$subitems" != "" ]; then											
+	  #echo ${subitems[@]}
-			 echo "adding ${subitems[@]}"
-	         $debug svn add ${subitems[@]}										#add all subitems
 	  
-	      	 for revpath in ${nonversionedpaths[@]}								#We need to revert sub-sub* dirs, since these havent been excluded
+	  for item in ${subitems[@]}
 	     do
-		       if test -d $revpath; then
+	     if test -d $item; then											#add or go recursive subitems
-		         if [ "${revpath:0:${#path}}" == "$path" ]; then
+		     $debug svn -N add $item									#NON recursively add the dir
-			 	   echo "Revert $revpath";
+		     subversionSingleStateDir $item
-			 	   $debug svn revert $revpath;
+		 else
-		         fi
+		 	 $debug svn add $item
 		 fi
 	  done
 
-		     $debug svn -m "" commit;													#Finally, we commit
+}  
+
+#
+#
+#
+#
+
+i=0
+for path in ${subversionedpaths[@]}
+do
+   if test -d $path; then														#if the dir doesnt exist, than we dont hav to do anything
+      cd $path;
+      
+      output=$($svnbin stat 2>&1 | grep "is not a working copy");
+      if [ "$output" != "" ] ; then       										#if the dir exists but is not yet an svn dir: checkout repos, if it doenst exits (is removed or something) than we dont do anything
+          $debug ${checkouts[$i]};
       fi
       
+      if [ "${subversionedpathsCommitBools[$i]}" = "true" ]; then				#Check if we need to commit this folder
+
+          echo "Entering $path"
+
+		  subversionSingleStateDir $path;
+		  
+		  cd $path																#now that everything is added we can commit
+	      $debug svn -m "" commit;												
 	   fi
       
       cd - &> /dev/null;
@@ -106,3 +136,7 @@ do
   let "i+=1"
 done
 
+
+
+
+

src/libexpr/primops.cc

@@ -470,7 +470,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
 				            	s = s + "/";
 				            
 				            //Remove the / at the beginning if it's there
-							if(s[0] == '/')
+							if(s[0] == '/' && s.length() != 1)
 				            	s = s.substr(1, s.length());
 				            
 				            if(s == stateRootRepos)
@@ -619,13 +619,17 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
 
     printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'") % drvName % drvPath);
 
-	/* TODO Write updated (no need to rebuild) state derivations to a special table, so they can be updated at build time */
+	/* Write updated (no need to rebuild) state derivations to the database, so they can be updated at build time */	
 	if(enableState && !disableState){
-		Path deriver = queryDeriver(noTxn, outPath);	//query deriver
+		if(store->isValidPath(outPath)){									//Only add when the path is already valid
+			Path deriver = queryDeriver(noTxn, outPath);			//query the deriver		
 			if(deriver != drvPath){
-			store->setUpdatedStateDerivation(drvPath, outPath);			
+				store->addUpdatedStateDerivation(drvPath, outPath);						
 			}
 		}
+		//TODO Also add when path is not already valid, which drv does it take at build time, the latest ... guess so .. so we dont need to add? 
+	}
+	
 
     /* Optimisation, but required in read-only mode! because in that
        case we don't actually write store expressions, so we can't

src/libstore/build.cc

@@ -2477,12 +2477,14 @@ void LocalStore::buildDerivations(const PathSet & drvPaths)
     startNest(nest, lvlDebug,
         format("building %1%") % showPaths(drvPaths));
 
+	//Just before we build, we resolve the multiple derivations linked to one store path issue, by choosing the latest derivation
+	store->updateAllStateDerivations();
+
     Worker worker;
 
     Goals goals;
     for (PathSet::const_iterator i = drvPaths.begin(); i != drvPaths.end(); ++i){
         goals.insert(worker.makeDerivationGoal(*i));
-        printMsg(lvlError, format("No component build, but state check: %1%") % *i);
     }
 
     worker.run(goals);

src/libstore/local-store.cc

@@ -1223,41 +1223,82 @@ PathSet LocalStore::getStateReferencesClosure(const Path & path)
 }
 
 
-//TODO
+void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
-void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
 {
 	Transaction txn(nixDB);
 
+	//Check wheter were not duplicating an entry, if so, than we wont have to do anything
+	//TODO
+	if(false){
+	
+		return;	
+	}
+
 	Strings data;
 	data.push_back(storepath);
-		
 	time_t timestamp;
   	time (&timestamp);
   	string timestamp_s = time_t2string(timestamp);
 	printMsg(lvlError, format("Adding new drv (%1%) to replace drv of old component (%2%) with timestamp: %3%") % newdrv % storepath % timestamp_s);
 	data.push_back(timestamp_s);										//create a timestamp to remember which one was last inserted
-							
     nixDB.setStrings(txn, dbUpdatedDerivations, newdrv, data);
     
-    //TODO check wheter were not duplicating an entry !!
-    
 	txn.commit();		
 }
 
-//TODO
+void LocalStore::addUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
-void LocalStore::setUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
 {
-	nix::setUpdatedStateDerivation(newdrv, storepath);
+	nix::addUpdatedStateDerivation(newdrv, storepath);
 }
 
-//TODO
+void updateAllStateDerivations()		  
-Path updateStateDerivation(const Path & storepath)		  
 {
 	Transaction txn(nixDB);
 	
-	Path drvPath = queryDeriver(txn, storepath);
+	Strings unique_paths;
-	printMsg(lvlError, format("Current DRV: %1%") % drvPath);
 	
+	Strings keys;
+    nixDB.enumTable(txn, dbUpdatedDerivations, keys);
+    for (Strings::iterator i = keys.begin(); i != keys.end(); ++i)
+    {
+		string drv_key = *i;	//the key is the derivation
+    	Strings data;
+		nixDB.queryStrings(txn, dbUpdatedDerivations, drv_key, data);
+	    string path = data.front();
+    	bool exists = false;
+    	 
+    	for (Strings::iterator j = unique_paths.begin(); j != unique_paths.end(); ++j)
+    	{
+			string unique_path = *j;
+    	 	
+    		if(path == unique_path)
+    			exists = true;
+    	}
+    	 
+    	if(!exists)
+    		unique_paths.push_back(path);
+    }
+	
+	for (Strings::iterator i = unique_paths.begin(); i != unique_paths.end(); ++i)
+    {
+    	string path = *i;
+    	printMsg(lvlError, format("Unique: %1%") % path);
+    	store->updateStateDerivation(txn, path);				//TODO replace store->
+    }
+    
+    txn.commit();
+}
+
+void LocalStore::updateAllStateDerivations()
+{
+	nix::updateAllStateDerivations();
+}
+
+
+void updateStateDerivation(const Transaction & txn, const Path & storepath)		  
+{
+	Path drvPath = queryDeriver(txn, storepath);
+	Path originalDerivation = drvPath;
 	Path newDerivation = drvPath;					//the new drv path first equals the old one until a new one is found 
 	int timestamp = 0;	
 	
@@ -1280,15 +1321,12 @@ Path updateStateDerivation(const Path & storepath)
 	    	string2Int(gettimestamp, gettimestamp_i);
 	    	
 	    	if(gettimestamp_i == timestamp)
-	    		throw Error(format("Error! Multiple changes at the same time of derivation: `%1%' with timestamp") % drv_key);		//TODO delete the mulitple changes ??
+	    		throw Error(format("Error! Multiple changes at store path %4% at the same time: derivations: `%1%' and `%2%' with timestamp `%3%'") % newDerivation % drv_key % timestamp % storepath);
 	    	
 	    	if(timestamp == 0 || gettimestamp_i > timestamp){		//we choose the new derivation as the latest submitted derivation
-	    		printMsg(lvlError, format("Replacing old drv (%1%) with new drv (%2%) with timestamp: %3%") % newDerivation % drv_key % gettimestamp_i);
+	    		//printMsg(lvlError, format("Replacing at store path %4% the old drv (%1%) with new drv (%2%) with timestamp: %3%") % newDerivation % drv_key % gettimestamp_i % storepath);
 	    		newDerivation = drv_key;
 	    		timestamp = gettimestamp_i;
-	    		
-	    		//Replace the old deriver link in the derivers database (TODO, and delete old deriver path????????)
-				setDeriver(txn, storepath, newDerivation);
 	    	}
 	    	
 	    	//Always Remove the old updatelink in the dbUpdatedDerivations
@@ -1296,14 +1334,21 @@ Path updateStateDerivation(const Path & storepath)
 	    } 
 	}
 	
-    txn.commit();
+	if(originalDerivation != newDerivation)		//only update if neccecary
-    return newDerivation;
+	{
+		//Replace the old deriver link in the derivers database (TODO, maybe delete old deriver path????????)
+		setDeriver(txn, storepath, newDerivation);	
+					
+		//Call the stateUpdate function for the new derivation? Yes since this function is called at build time
+		printMsg(lvlError, format("Calling new state drv %1% for storepath %2%") % newDerivation % storepath);
+		//TODO check wheter we update before or after ??
+		//TODO
+	}
 }
 
-//TODO
+void LocalStore::updateStateDerivation(const Transaction & txn, const Path & storepath)
-Path LocalStore::updateStateDerivation(const Path & storepath)
 {
-	return nix::updateStateDerivation(storepath);
+	nix::updateStateDerivation(txn, storepath);
 }
 
 

src/libstore/local-store.hh

@@ -84,9 +84,11 @@ public:
 	
 	PathSet getStateReferencesClosure(const Path & path);
 	
-	void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath);
+	void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath);
 
-	Path updateStateDerivation(const Path & storepath);
+	void updateStateDerivation(const Transaction & txn, const Path & storepath);
+	
+	void updateAllStateDerivations();
 	
 };
 

src/libstore/remote-store.cc

@@ -403,16 +403,21 @@ PathSet RemoteStore::getStateReferencesClosure(const Path & path)
 }
 
 //TODO
-void RemoteStore::setUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
+void RemoteStore::addUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
 {
 	
 }
 
 //TODO
-Path RemoteStore::updateStateDerivation(const Path & storepath)
+void RemoteStore::updateStateDerivation(const Transaction & txn, const Path & storepath)
 {
-	Path p;
+	
-	return p;
+}
+
+//TODO
+void RemoteStore::updateAllStateDerivations()
+{
+	
 }
 
 

src/libstore/remote-store.hh

@@ -72,9 +72,11 @@ public:
 	
 	PathSet getStateReferencesClosure(const Path & path);
 	
-	void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath);
+	void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath);
 
-	Path updateStateDerivation(const Path & storepath);
+	void updateStateDerivation(const Transaction & txn, const Path & storepath);
+	
+	void updateAllStateDerivations();
 	
     
 private:

src/libstore/store-api.hh

@@ -9,6 +9,7 @@
 #include "hash.hh"
 #include "serialise.hh"
 #include "derivations.hh"
+#include "db.hh"
 
 
 namespace nix {
@@ -194,10 +195,13 @@ public:
 	virtual PathSet getStateReferencesClosure(const Path & path) = 0;
 	
 	/* TODO */	
-	virtual void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath) = 0;
+	virtual void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath) = 0;
 
 	/* TODO */	
-	virtual Path updateStateDerivation(const Path & storepath) = 0;
+	virtual void updateStateDerivation(const Transaction & txn, const Path & storepath) = 0;
+	
+	/* TODO */
+	virtual void updateAllStateDerivations() = 0;
 	
 };
 

src/libstore/store-state.cc

@@ -16,9 +16,12 @@
 namespace nix {
 
 
-void dsfsdfas()
+void updatedStateDerivation(Path storePath)
 {
+	//Remove the old .svn folders
 		
+	//Create new repositorys, or use existing...
+	//createStateDirs already does that ...
 }
 
 

src/libutil/util.cc

@@ -1016,7 +1016,7 @@ string trim(const string & s) {
 
 void executeAndPrintShellCommand(const string & command, const string & commandName)
 {
-	string tempoutput = "svnoutput.txt";
+	string tempoutput = "/tmp/svnoutput.txt";
 	string newcommand = command + " &> " + tempoutput;		//the  &> sends also stderr to stdout
 
 	int kidstatus, deadpid;

src/nix-state/nix-state.cc

@@ -102,7 +102,9 @@ static void opRunComponent(Strings opFlags, Strings opArgs)
     
     //Check if component is a state component !!!
     
-	//Wait for locks?
+    //Check for locks ...
+	//add locks ... ?
+	//svn lock ... ?
 	
     //******************* Run the component
     //TODO
@@ -156,7 +158,10 @@ static void opRunComponent(Strings opFlags, Strings opArgs)
 		DerivationStateOutputDir d = i->second;
 
 		string thisdir = d.path;
+		
 		string fullstatedir = statePath + "/" + thisdir;
+		if(thisdir == "/")									//exception for the root dir
+			fullstatedir = statePath + "/";				
 		Path statePath = fullstatedir;					//TODO call coerce function
 
 		if(d.type == "none"){
@@ -229,11 +234,13 @@ void run(Strings args)
     Strings opFlags, opArgs;
     Operation op = 0;
 
-	/* test */
+	/* test *
 	store = openStore();
-	Path p = "/nix/store/l569q3a2cfx834mcf3vhwczjgbaljnp7-hellohardcodedstateworld-1.0";
+	Path p = "/nix/store/l569q3a2cfx834mcf3vhwczjgbaljnp7-hellohardcodedstateworld-1.0";										//
-	store->setUpdatedStateDerivation("/nix/store/63xcbrk3v5nbn9qla7rwnx6rvz3iqm5l-hellohardcodedstateworld-1.0.drv", p);
+	store->addUpdatedStateDerivation("/nix/store/63xcbrk3v5nbn9qla7rwnx6rvz3iqm5l-hellohardcodedstateworld-1.0.drv", p);		//
-	store->updateStateDerivation(p);
+	Path p2 = "/nix/store/4ycq45hsgc8yaj4vwafx3lgd473jaqwg-hellohardcodedstateworld-1.0";
+	store->addUpdatedStateDerivation("/nix/store/s6wggk924jx0gcb0l29ra4g9fxa3b4pp-hellohardcodedstateworld-1.0.drv", p2);		//
+	store->updateAllStateDerivations();
 	return;
 	/* test */
 
@@ -261,6 +268,14 @@ void run(Strings args)
 		
 		TODO update getDerivation in nix-store to handle state indentifiers
 		
+		--update state drv
+		
+		--revert-to-state	(recursive revert...)
+		
+		--delete state?
+		
+		--
+		
         */
 
         else