Before adjusting queryReferences/Referrers to handle shared state paths

src/libexpr/primops.cc

@@ -417,11 +417,12 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
   	//state vars
     bool enableState = false;		//We dont do state by default, but if a user defines stateDirs for example, than this becomes true.
     bool disableState = false;		//Becomes true if the user explicitly says: no state
-    string shareState = "none";
+    string shareType = "none";
     string syncState = "none";
     string stateIdentifier = "";
     bool createDirsBeforeInstall = false;
     string runtimeStateParamters = "";
+    string sharedState = "";
     vector<DerivationStateOutputDir> stateDirs;
 
     for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i) {
@@ -520,12 +521,13 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
                     drv.solidStateDeps.insert(s);
                 }
             }
-   	        else if(key == "shareState") { shareState = coerceToString(state, value, context, true); }
+   	        else if(key == "shareType") { shareType = coerceToString(state, value, context, true); }
    	        else if(key == "synchronization") { syncState = coerceToString(state, value, context, true); }
    	        else if(key == "disableState") { disableState = evalBool(state, value); }
             else if(key == "identifier"){ stateIdentifier = coerceToString(state, value, context, true); }
             else if(key == "createDirsBeforeInstall"){ createDirsBeforeInstall = evalBool(state, value); }
             else if(key == "runtimeStateParamters"){ runtimeStateParamters = coerceToString(state, value, context, true); }
+            else if(key == "shareStateFrom"){ sharedState = coerceToString(state, value, context, true);  }
             
             /* All other attributes are passed to the builder through
                the environment. */
@@ -614,7 +616,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
 	if(enableState && !disableState){    
 		if(runtimeStateParamters == ""){
 			string enableStateS = bool2string("true");
-			drv.stateOutputs["state"] = DerivationStateOutput("", "", "", "", stateIdentifier, enableStateS, "", "", "", runtimeStateParamters, getCallingUserName(), false);
+			drv.stateOutputs["state"] = DerivationStateOutput("", "", "", "", stateIdentifier, enableStateS, "", "", "", runtimeStateParamters, getCallingUserName(), "", false);
 		}	
 	}
         
@@ -649,7 +651,8 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
 
     	string enableStateS = bool2string("true");
     	string createDirsBeforeInstallS = bool2string(createDirsBeforeInstall);
-    	drv.stateOutputs["state"] = DerivationStateOutput(stateOutPath, printHash(componentHash), outputHashAlgo, outputHash, stateIdentifier, enableStateS, shareState, syncState, createDirsBeforeInstallS, runtimeStateParamters, getCallingUserName());
+    	drv.stateOutputs["state"] = DerivationStateOutput(stateOutPath, printHash(componentHash), outputHashAlgo, outputHash, stateIdentifier, enableStateS, 
+    	                                                  shareType, syncState, createDirsBeforeInstallS, runtimeStateParamters, getCallingUserName(), sharedState);
     	
     	for(vector<DerivationStateOutputDir>::iterator i = stateDirs.begin(); i != stateDirs.end(); ++i)
     		drv.stateOutputDirs[(*i).path] = *(i);

src/libstore/build.cc

@@ -1754,12 +1754,23 @@ void DerivationGoal::computeClosure()
      * [scan for and state references and component references in the state path]	//3,4
      */
 	if(isStateDrvTxn(noTxn, drv)){		//TODO 
+		
+		Path sharedState = drv.stateOutputs.find("state")->second.sharedState;
+		if(sharedState != ""){
+			if (!store->isValidStatePath(sharedState))
+				throw BuildError(format("sharedState path `%1%', is not a valid state path") % sharedState); 
+			
+			//We dont need to scan for state references since at the query to the state path we give the results of the linked-to path
+		}
+		else
+		{
 			Path statePath = drv.stateOutputs.find("state")->second.statepath;
 			printMsg(lvlTalkative, format("scanning for component and state references inside `%1%'") % statePath);
 		
 			state_references = scanForReferences(statePath, allPaths);
 			state_stateReferences = scanForStateReferences(statePath, allStatePaths);
 		}
+	}
 	
     /* Register each output path as valid, and register the sets of
        paths referenced by each of them.  This is wrapped in one
@@ -1789,7 +1800,6 @@ void DerivationGoal::computeClosure()
     //Register the state path valid
     if(isStateDrvTxn(txn, drv))
     {
-		//TODO ONLY CALL THIS FUNCTION ON A NON-SHARED STATE PATH!!!!!!!!!!!	(why?)
 		Path statePath = drv.stateOutputs.find("state")->second.statepath;		
 		
     	registerValidPath(txn,    	
@@ -1797,7 +1807,7 @@ void DerivationGoal::computeClosure()
     		Hash(),										//emtpy hash
     		state_references,
     		state_stateReferences,
-    		drvPath, 0);							//TODO !!!!!!!!!!!!
+    		drvPath, 0);
 
     	//Commit state
 		commitStatePathTxn(txn, statePath);
@@ -1805,6 +1815,23 @@ void DerivationGoal::computeClosure()
 		//Set first revision (if we committed something)
 		if(readRevisionNumber(statePath) == 1)
 			updateRevisionsRecursivelyTxn(txn, statePath);
+
+		//Shared state
+    	Path sharedState = drv.stateOutputs.find("state")->second.sharedState;
+		if(sharedState != ""){
+			//Remove state path
+			deletePathWrapped(statePath);
+			
+			//Symlink link to the share path
+			Strings p_args;
+			p_args.push_back("-sf");
+			p_args.push_back(sharedState);
+			p_args.push_back(statePath);
+			runProgram_AndPrintOutput("ln", true, p_args, "ln");	//run
+			
+			//Set in database
+			setSharedStateTxn(txn, statePath, sharedState);
+		}
     }	    
     
     txn.commit();

src/libstore/derivations-ast.def

@@ -4,7 +4,7 @@ Derive | ATermList ATermList ATermList ATermList ATermList ATermList string stri
 | string string | ATerm | EnvBinding |
 | string ATermList | ATerm | DerivationInput |
 | string string string string | ATerm | DerivationOutput |
-| string string string string string string string string string string string string | ATerm | DerivationStateOutput |
+| string string string string string string string string string string string string string | ATerm | DerivationStateOutput |
 | string string string | ATerm | DerivationStateOutputDir |
 
 #We use DeriveWithOutState to create derivations that dont use state, and thus dont have the stateDerivationStateOutput and DerivationStateOutputDir in their derivation

src/libstore/derivations.cc

@@ -93,8 +93,8 @@ Derivation parseDerivation(ATerm t)
     {
 	    //parse state part
 	    for (ATermIterator i(stateOuts); i; ++i) {
-	        ATerm id, statepath, componentHash, hashAlgo, hash, stateIdentifier, enabled, shared, synchronization, createDirsBeforeInstall, runtimeStateParamters, username;
+	        ATerm id, statepath, componentHash, hashAlgo, hash, stateIdentifier, enabled, shareType, synchronization, createDirsBeforeInstall, runtimeStateParamters, username, sharedState;
-	        if (!matchDerivationStateOutput(*i, id, statepath, componentHash, hashAlgo, hash, stateIdentifier, enabled, shared, synchronization, createDirsBeforeInstall, runtimeStateParamters, username))
+	        if (!matchDerivationStateOutput(*i, id, statepath, componentHash, hashAlgo, hash, stateIdentifier, enabled, shareType, synchronization, createDirsBeforeInstall, runtimeStateParamters, username, sharedState))
 	            throwBadDrv(t);
 	        DerivationStateOutput stateOut;
 	        stateOut.statepath = aterm2String(statepath);
@@ -104,11 +104,12 @@ Derivation parseDerivation(ATerm t)
 	        stateOut.hash = aterm2String(hash);
 	        stateOut.stateIdentifier = aterm2String(stateIdentifier);
 	        stateOut.enabled = aterm2String(enabled);
-	        stateOut.shared = aterm2String(shared);
+	        stateOut.shareType = aterm2String(shareType);
 	        stateOut.synchronization = aterm2String(synchronization);
 	        stateOut.createDirsBeforeInstall = aterm2String(createDirsBeforeInstall);
 	        stateOut.runtimeStateParamters = aterm2String(runtimeStateParamters);
 	        stateOut.username = aterm2String(username);
+	        stateOut.sharedState = aterm2String(sharedState);  
 	        drv.stateOutputs[aterm2String(id)] = stateOut;
 	    }
 
@@ -195,11 +196,12 @@ ATerm unparseDerivation(const Derivation & drv)
                 toATerm(i->second.hash),
                 toATerm(i->second.stateIdentifier),
                 toATerm(i->second.enabled),
-                toATerm(i->second.shared),
+                toATerm(i->second.shareType),
                 toATerm(i->second.synchronization),
                 toATerm(i->second.createDirsBeforeInstall),
                 toATerm(i->second.runtimeStateParamters),
-                toATerm(i->second.username)
+                toATerm(i->second.username),
+                toATerm(i->second.sharedState)
                 ));
     }
                 

src/libstore/derivations.hh

@@ -42,7 +42,7 @@ struct DerivationStateOutput
     string hash;
     string stateIdentifier;				//the identifier
     string enabled;						//enable or disable state
-    string shared;						//none, full, group
+    string shareType;						//none, full, group
     string synchronization;				//none (no locks), exclusive-lock, recursive-exclusive-lock
     
     string commitReferences;			//TODO none, direct, recursive-all
@@ -53,16 +53,18 @@ struct DerivationStateOutput
     
     string username;	
     
+    string sharedState;					//Path to share state From		
+    
     DerivationStateOutput()
     {
     }
     
     //TODO add const ??
-    DerivationStateOutput(Path statepath, string componentHash, string hashAlgo, string hash, string stateIdentifier, string enabled, string shared, string synchronization, string createDirsBeforeInstall, string runtimeStateParamters, string username, bool check=true)
+    DerivationStateOutput(Path statepath, string componentHash, string hashAlgo, string hash, string stateIdentifier, string enabled, string shareType, string synchronization, string createDirsBeforeInstall, string runtimeStateParamters, string username, string sharedState, bool check=true)
     {
         if(check){
-	        if(shared != "none" && shared != "full" && shared != "group")
+	        if(shareType != "none" && shareType != "full" && shareType != "group")
-	        	throw Error(format("shared '%1%' is not a correct type") % shared);
+	        	throw Error(format("shareType '%1%' is not a correct type") % shareType);
 	        if(synchronization != "none" && synchronization != "exclusive-lock" && synchronization != "recursive-exclusive-lock")
 	        	throw Error(format("synchronization '%1%' is not a correct type") % synchronization);
 	        if(username == "")
@@ -79,11 +81,12 @@ struct DerivationStateOutput
         this->hash = hash;
         this->stateIdentifier = stateIdentifier;
         this->enabled = enabled;
-        this->shared = shared;
+        this->shareType = shareType;
         this->synchronization = synchronization;
         this->createDirsBeforeInstall = createDirsBeforeInstall;
         this->runtimeStateParamters = runtimeStateParamters;
         this->username = username;
+        this->sharedState = sharedState;
     }
     
     bool getEnabled(){
@@ -104,11 +107,13 @@ struct DerivationStateOutput
         //this->hash;							//Clear this one?
         //this->stateIdentifier;				//Changes the statepath directly
         this->enabled = "";
-        this->shared = "";
+        this->shareType = "";
         this->synchronization = "";
         this->createDirsBeforeInstall = "";
         this->runtimeStateParamters = "";
         //this->username;						//Changes the statepath directly
+        this->sharedState = "";
+         
     }
 };
 

src/libstore/local-store.cc

@@ -131,6 +131,12 @@ static TableId dbStateInfo = 0;
 */
 static TableId dbStateRevisions = 0;
 
+/* dbSharedState :: Path -> Path
+ * 
+ * Lists all paths that are shared with other paths 
+ */
+static TableId dbSharedState = 0;
+
 bool Substitute::operator == (const Substitute & sub) const
 {
     return program == sub.program
@@ -205,6 +211,7 @@ LocalStore::LocalStore(bool reserveSpace)
 	dbStateComponentReferrers = nixDB.openTable("referrers_s_c", true);
 	dbStateStateReferrers = nixDB.openTable("referrers_s_s", true);
 	dbStateRevisions = nixDB.openTable("staterevisions");
+	dbSharedState = nixDB.openTable("sharedState");
 
 	dbSolidStateReferences = nixDB.openTable("references_solid_c_s");	/* The contents of this table is included in references_c_s */
 	
@@ -406,6 +413,7 @@ static string stripPrefix(const string & prefix, const string & s)
 
 //TODO move the code of get(State)Referrers into query variant ..... !!!!!!!!!!!!!!!!!!!!!!!!!!!!???  
 
+/********************/
 
 static PathSet getReferrers(const Transaction & txn, const Path & store_or_statePath, const int revision)
 {
@@ -631,6 +639,8 @@ void LocalStore::queryStateReferrers(const Path & storePath, PathSet & stateRefe
     nix::queryStateReferrersTxn(noTxn, storePath, stateReferrers, revision);
 }
 
+/********************/
+
 
 void setDeriver(const Transaction & txn, const Path & storePath, const Path & deriver)
 {
@@ -1731,6 +1741,41 @@ bool querySolidStateReferencesTxn(const Transaction & txn, const Path & statePat
 	return notempty;
 }
 
+void setSharedStateTxn(const Transaction & txn, const Path & statePath, const Path & shared_with)
+{
+	//Remove earlier entries
+	nixDB.delPair(txn, dbSharedState, statePath);
+
+	//Set new entry
+	nixDB.setString(txn, dbSharedState, statePath, shared_with);
+}
+
+bool querySharedStateTxn(const Transaction & txn, const Path & statePath, Path & shared_with)
+{
+	return nixDB.queryString(txn, dbSharedState, statePath, shared_with);
+}
+
+PathSet toNonSharedPathSetTxn(const Transaction & txn, const PathSet & statePaths)
+{
+	PathSet real_statePaths;
+
+	//we loop over all paths in the list
+	for (PathSet::const_iterator i = statePaths.begin(); i != statePaths.end(); ++i){
+		Path sharedPath;
+		Path checkPath = *i;
+		real_statePaths.insert(checkPath);
+		
+		//for each path we do querySharedStateTxn until there the current path is not a shared path anymore  
+		while(querySharedStateTxn(txn, checkPath, sharedPath)){
+			real_statePaths.erase(checkPath);
+			real_statePaths.insert(sharedPath);
+			checkPath = sharedPath;
+		}
+	}
+		
+	return real_statePaths;
+}
+	
 /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */
 static void upgradeStore07()
 {

src/libstore/local-store.hh

@@ -243,6 +243,9 @@ bool isValidStatePathTxn(const Transaction & txn, const Path & path);
 void setSolidStateReferencesTxn(const Transaction & txn, const Path & statePath, const PathSet & paths);
 bool querySolidStateReferencesTxn(const Transaction & txn, const Path & statePath, PathSet & paths);
 
+void setSharedStateTxn(const Transaction & txn, const Path & statePath, const Path & shared_with);
+PathSet toNonSharedPathSetTxn(const Transaction & txn, const PathSet & statePaths);
+
 }
 
 

src/nix-state/nix-state.cc

@@ -249,18 +249,6 @@ static void revertToRevision(Strings opFlags, Strings opArgs)
     RevisionNumbersSet getRivisions;
 	bool b = store->queryStateRevisions(statePath, getRivisions, revision_arg);
     
-    //Sort the statePaths from all drvs
-	//map<Path, string> state_repos;
-	//vector<Path> sorted_paths;
-    //for (PathSet::iterator d = statePaths.begin(); d != statePaths.end(); ++d){
-       	
-		//state_repos[statePath] = repos; 
-//		sorted_paths.push_back(statePath);
-	//}
-	//sort(sorted_paths.begin(), sorted_paths.end());	
-	
-	//string repos = 
-	
 	//Revert each statePath in the list
 	for (RevisionNumbersSet::iterator i = getRivisions.begin(); i != getRivisions.end(); ++i){
 		Path statePath = (*i).first;
@@ -413,6 +401,9 @@ static void opRunComponent(Strings opFlags, Strings opArgs)
 	store->storePathRequisites(root_componentPath, false, statePaths, false, true, -1);
 	statePaths.insert(root_statePath);
     
+    //Replace all shared paths in the set for their real paths 
+    statePaths = toNonSharedPathSetTxn(noTxn, statePaths);
+    
     //TODO maybe also scan the parameters for state or component hashes?
     //program_args