before removing referrer code

src/libstore/db.cc

@@ -449,13 +449,13 @@ void Database::enumTable(const Transaction & txn, TableId table,
 
 /* State specific db functions */
 
-Path Database::makeStatePathRevision(const Path & statePath, const int revision)
+Path Database::mergeToDBKey(const Path & statePath, const int revision)
 {
 	string prefix = "-KEY-";
 	return statePath + prefix + int2String(revision);
 }
 
-void Database::splitStatePathRevision(const Path & revisionedStatePath, Path & statePath, int & revision)
+void Database::splitDBKey(const Path & revisionedStatePath, Path & statePath, int & revision)
 {
 	string prefix = "-KEY-";
 	
@@ -503,7 +503,7 @@ bool Database::lookupHighestRevivison(const Strings & keys, const Path & statePa
 		//printMsg(lvlError, format("'%1%' - '%2%'") % *i % statePath);
 		Path getStatePath;
 		int getRevision;
-		splitStatePathRevision(*i, getStatePath, getRevision);
+		splitDBKey(*i, getStatePath, getRevision);
 		if(getRevision > highestRev){
 		
 			if(lowerthan != -1){
@@ -518,19 +518,19 @@ bool Database::lookupHighestRevivison(const Strings & keys, const Path & statePa
 	if(highestRev == -1)	//no records found
 		return false;
 	
-	key = makeStatePathRevision(statePath, highestRev);		//final key that matches revision + statePath
+	key = mergeToDBKey(statePath, highestRev);		//final key that matches revision + statePath
 	return true;	
 }
 
 bool Database::revisionToTimeStamp(const Transaction & txn, TableId revisions_table, const Path & statePath, const int revision, int & timestamp)
 {
-	string key = makeStatePathRevision(statePath, revision);
+	string key = mergeToDBKey(statePath, revision);
 	Strings references;
 	bool notempty = queryStrings(txn, revisions_table, key, references);
 	
 	if(notempty){	
 		Path empty; 
-		splitStatePathRevision(*(references.begin()), empty, timestamp);	//extract the timestamp
+		splitDBKey(*(references.begin()), empty, timestamp);	//extract the timestamp
 		//printMsg(lvlError, format("PRINT '%1%'") % timestamp);
 		return true;
 	}
@@ -561,10 +561,15 @@ void Database::setStateReferences(const Transaction & txn, TableId references_ta
 		printMsg(lvlError, format("Warning: The timestamp '%1%' already exists for set-references/referrers of path '%2%' with db '%3%'") % timestamp % statePath % references_table);
 	
 	//Create the key
-	string key = makeStatePathRevision(statePath, timestamp);
+	string key = mergeToDBKey(statePath, timestamp);
+	
+	
+	printMsg(lvlError, format("Set references '%1%'") % key);
+	for (Strings::const_iterator i = references.begin(); i != references.end(); ++i)
+		printMsg(lvlError, format("reference '%1%'") % *i);
 	
 	//Insert	
-	setStrings(txn, references_table, key, references);
+	setStrings(txn, references_table, key, references, false);				//The false makes sure also empty references are set
 }
 
 bool Database::queryStateReferences(const Transaction & txn, TableId references_table, TableId revisions_table,
@@ -593,12 +598,12 @@ bool Database::queryStateReferences(const Transaction & txn, TableId references_
 	}
 	
 	//If a specific key is given: check if this timestamp exists key in the table
-	key = makeStatePathRevision(statePath, timestamp);
+	key = mergeToDBKey(statePath, timestamp);
 	bool found = false;
 	for (Strings::const_iterator i = keys.begin(); i != keys.end(); ++i) {
 		if(key == *i){
 			found = true;
-			key = makeStatePathRevision(statePath, timestamp);
+			key = mergeToDBKey(statePath, timestamp);
 		}
 	}
 	
@@ -627,6 +632,55 @@ bool Database::queryStateReferrers(const Transaction & txn, TableId referrers_ta
 	return queryStateReferences(txn, referrers_table, revisions_table, statePath, referrers, revision, timestamp);
 }
 
+ //PRIVATE !!!!!!
+//Update the referrers of some path because the store_or_statePath got new references
+void Database::updateReferredPath(const Transaction & txn, TableId revisions_table, TableId referrers_X_s_table,
+	const Path & referred_state_or_store_Path, const Path & statePath, const int revision)
+{
+	
+	//referred_state_or_store_Path --> statePath
+	
+	//you dont know what referred_state_or_store_Path is !!!!!!!!!!!! add a bool !!!!!!
+	
+	if(revision != -1){
+		//Get timestamp of revision
+		int timestamp;
+		bool succeed = revisionToTimeStamp(txn, revisions_table, referred_state_or_store_Path, revision, timestamp);
+		if(!succeed)
+			throw Error(format("Couldnt find timestamp for revision '%1%' at update referrers mapping for '%2%'") % revision % referred_state_or_store_Path);
+		
+		//remove all dynamic (that have a ts) referrers of that revision first (except if it is the latest, eg. -1)
+		delPair(txn, referrers_X_s_table, mergeToDBKey(referred_state_or_store_Path, timestamp));
+		
+	}
+		
+	//now get the referrers at timestamp (or just the latest if revision is -1)	
+	PathSet referrers;
+	//TODO
+	
+	//if( ! state)
+	//getReferrers...............	
+	//dbStateComponentReferrers
+	//else
+	//getStateReferrers...............
+	//dbStateStateReferrers
+	
+	
+	//string kkkk = nixDB.mergeToDBKey(store_or_statePath, revision);
+    	
+ 	//Add store_or_statePath in .... if nessacary (if in references)
+ 	//TODO
+ 	//referredPath - TS --> store_or_statePath ADD
+ 			
+ 	//Set Remove store_or_statePath in .... if nessacary (if not in references)
+	//TODO
+	//referredPath - TS --> store_or_statePath DEL
+	
+	
+	//set the new referrers of revision (-1 insert as a new timestamp)
+	//nixDB.setStateReferrers(txn, table, dbStateRevisions, referredPath, ........, revision);
+}
+
 void Database::setStateRevisions(const Transaction & txn, TableId revisions_table, TableId snapshots_table,
    	const RevisionClosure & revisions)
 {
@@ -634,7 +688,7 @@ void Database::setStateRevisions(const Transaction & txn, TableId revisions_tabl
 	
 	//Insert all ss_epochs into snapshots_table with the current ts.
 	for (RevisionClosure::const_iterator i = revisions.begin(); i != revisions.end(); ++i){
-		string key = makeStatePathRevision((*i).first, ts);
+		string key = mergeToDBKey((*i).first, ts);
 		Strings data;
 		//the map<> takes care of the sorting on the Path
 		for (Snapshots::const_iterator j = (*i).second.begin(); j != (*i).second.end(); ++j)
@@ -647,7 +701,7 @@ void Database::setStateRevisions(const Transaction & txn, TableId revisions_tabl
 		Path statePath = (*i).first;
 		
 		int revision = getNewRevisionNumber(txn, revisions_table, statePath);	//get a new revision number
-		string key = makeStatePathRevision(statePath, revision);
+		string key = mergeToDBKey(statePath, revision);
 		
 		//get all its requisites
 		PathSet statePath_references;
@@ -657,8 +711,9 @@ void Database::setStateRevisions(const Transaction & txn, TableId revisions_tabl
 		//save in db
 		Strings data;
 		for (PathSet::const_iterator j = statePath_references.begin(); j != statePath_references.end(); ++j)
-			data.push_back(makeStatePathRevision(*j, ts));		
+			data.push_back(mergeToDBKey(*j, ts));		
-		setStrings(txn, revisions_table, key, data);
+		
+		setStrings(txn, revisions_table, key, data, false);				//The false makes sure also empty revisions are set
 	}
 }   
 
@@ -675,7 +730,7 @@ bool Database::queryStateRevisions(const Transaction & txn, TableId revisions_ta
 			return false;
 	}
 	else
-		key = makeStatePathRevision(statePath, root_revision);
+		key = mergeToDBKey(statePath, root_revision);
 	
 	//Get references pointing to snapshots_table from revisions_table with root_revision
 	Strings statePaths;
@@ -689,7 +744,7 @@ bool Database::queryStateRevisions(const Transaction & txn, TableId revisions_ta
 		
 		Path getStatePath;
 		int getTimestamp;
-		splitStatePathRevision(*i, getStatePath, getTimestamp);
+		splitDBKey(*i, getStatePath, getTimestamp);
 		
 		//query state versioined directorys/files
 		vector<Path> sortedPaths;
@@ -742,7 +797,7 @@ bool Database::queryAvailableStateRevisions(const Transaction & txn, TableId rev
 
 		Path getStatePath;
 		int getRevision;
-		splitStatePathRevision(*i, getStatePath, getRevision);
+		splitDBKey(*i, getStatePath, getRevision);
 		revisions.push_back(getRevision);
 	}
     

src/libstore/db.hh

@@ -100,10 +100,10 @@ public:
         Strings & keys, const string & keyPrefix = "");
     
     /* TODO */
-    Path makeStatePathRevision(const Path & statePath, const int revision);
+    Path mergeToDBKey(const Path & statePath, const int revision);
     
     /* TODO */
-    void splitStatePathRevision(const Path & revisionedStatePath, Path & statePath, int & revision);
+    void splitDBKey(const Path & revisionedStatePath, Path & statePath, int & revision);
     
     /* Set the stateReferences for a specific revision (and older until the next higher revision number in the table) */    
     void setStateReferences(const Transaction & txn, TableId references_table, TableId revisions_table,
@@ -121,6 +121,10 @@ public:
     bool queryStateReferrers(const Transaction & txn, TableId referrers_table, TableId revisions_table,
     	const Path & statePath, Strings & referrers, int revision = -1, int timestamp = -1);
     
+    /* TODO */
+    void updateReferredPath(const Transaction & txn, TableId revisions_table, TableId referrers_X_s_table,
+    	const Path & referred_state_or_store_Path, const Path & statePath, const int revision);
+     
     /* Set the revision number of the statePath and the revision numbers of all state paths in the references closure */
     void setStateRevisions(const Transaction & txn, TableId revisions_table, TableId snapshots_table,
     	const RevisionClosure & revisions);

src/libstore/local-store.cc

@@ -418,6 +418,8 @@ static string stripPrefix(const string & prefix, const string & s)
     return string(s, prefix.size() + 1);
 }
 
+
+
 void setReferences(const Transaction & txn, const Path & store_or_statePath,
     const PathSet & references, const PathSet & stateReferences, const int revision)
 {
@@ -445,12 +447,12 @@ void setReferences(const Transaction & txn, const Path & store_or_statePath,
 
     	PathSet oldReferences = PathSet(oldReferences_c_c.begin(), oldReferences_c_c.end());
     	PathSet oldStateReferences = PathSet(oldReferences_c_s.begin(), oldReferences_c_s.end());
-    	if (oldReferences == references && oldStateReferences == stateReferences) return;
+    	if (oldReferences == references && oldStateReferences == stateReferences) return;						//watch out we way need to set the referrers.... (at a ts) 
     	
     	nixDB.setStrings(txn, dbComponentComponentReferences, store_or_statePath, Paths(references.begin(), references.end()));
 		nixDB.setStrings(txn, dbComponentStateReferences, store_or_statePath, Paths(stateReferences.begin(), stateReferences.end()));
 	
-		//Handle referrers ......
+		//Handle referrers like the old way, because references (and so referrers) referrers never change (So these do not have a timestamp)
 		   		
    		//dbComponentComponentReferrers
    		//dbComponentStateReferrers
@@ -477,53 +479,27 @@ void setReferences(const Transaction & txn, const Path & store_or_statePath,
 
 		PathSet oldReferences = PathSet(oldStateReferences_s_c.begin(), oldStateReferences_s_c.end());
     	PathSet oldStateReferences = PathSet(oldStateReferences_s_s.begin(), oldStateReferences_s_s.end());
-		//if (oldReferences == references && oldStateReferences == stateReferences) return;								//TODO Turn on ????????????
 
 		//set the references of revision (-1 insert as a new timestamp)
     	nixDB.setStateReferences(txn, dbStateComponentReferences, dbStateRevisions, store_or_statePath, Paths(references.begin(), references.end()), revision);
 		nixDB.setStateReferences(txn, dbStateStateReferences, dbStateRevisions, store_or_statePath, Paths(stateReferences.begin(), stateReferences.end()), revision);
 
-
 		//REFERRERS
 
    		//for all references old and new, state and component
-   		PathSet all = pathSets_union(pathSets_union(references, oldReferences), pathSets_union(oldStateReferences, stateReferences));		//TODO SPLIT in state and non state
+   		PathSet all_references = pathSets_union(references, oldReferences);
-   		for (PathSet::const_iterator i = all.begin(); i != all.end(); ++i){
+		PathSet all_stateReferences = pathSets_union(oldStateReferences, stateReferences);
    		
-   			Path referredPath = *i;		//Path referred to by the statePath
+   		for (PathSet::const_iterator i = all_references.begin(); i != all_references.end(); ++i)
+   			nixDB.updateReferredPath(txn, dbStateRevisions, dbComponentStateReferrers, *i, store_or_statePath, revision);
    		
-   			//query the refererrers of revision (-1 is query the latest refererrers)
+   		for (PathSet::const_iterator i = all_stateReferences.begin(); i != all_stateReferences.end(); ++i)
-			Paths referrers_s_c;
+   			nixDB.updateReferredPath(txn, dbStateRevisions, dbStateStateReferrers, *i, store_or_statePath, revision);
-			Paths referrers_s_s;
-			nixDB.queryStateReferrers(txn, dbStateComponentReferrers, dbStateRevisions, referredPath, referrers_s_c, revision);		//may contain other referred by paths 
-			nixDB.queryStateReferrers(txn, dbStateStateReferrers, dbStateRevisions, referredPath, referrers_s_s, revision);
-	   		PathSet oldReferrers = PathSet(referrers_s_c.begin(), referrers_s_c.end());
-	    	PathSet oldStateReferrers = PathSet(referrers_s_s.begin(), referrers_s_s.end());
    		
-   			
-   			//Add store_or_statePath in  if nessacary (if in references)
-   			//TODO
-   			
-   			//Remove store_or_statePath if nessacary (if not in references)
-   			//TODO
-   			
-   			//set the new referrers of revision (-1 insert as a new timestamp)
-    		//nixDB.setStateReferrers(txn, dbStateComponentReferences, dbStateRevisions, referredPath, ........, revision);
-			//nixDB.setStateReferrers(txn, dbStateStateReferences, dbStateRevisions, referredPath, ......, revision);
-   			
-   		}
     }
     else
     	throw Error(format("Path '%1%' is not a valid component or state path") % store_or_statePath);
     
-   	//store_or_statePath must be postfixed in case it is a statePath 
-    Path store_or_statePath2 = store_or_statePath;
-    if(isRealisableStatePath(txn, store_or_statePath))
-   		store_or_statePath2 = nixDB.makeStatePathRevision(store_or_statePath, revision);								//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! THIS OK ???
-																														//NO ???? SINCE THE KEYS SHOULD BE TIMESTAMPED ??
-																														//WE NEEED TO inlcude the revision !!
-        
-
 }
 
 void queryReferencesTxn(const Transaction & txn, const Path & store_or_statePath, PathSet & references, const int revision, int timestamp)
@@ -585,6 +561,7 @@ void LocalStore::queryAllReferences(const Path & path, PathSet & allReferences,
 
 static PathSet getXReferrers(const Transaction & txn, const Path & store_or_statePath, const int revision, const TableId & table)
 {
+    /*
     if(isValidPathTxn(txn, store_or_statePath)){
 	    PathSet referrers;
     	Strings keys;
@@ -602,6 +579,70 @@ static PathSet getXReferrers(const Transaction & txn, const Path & store_or_stat
     }
     else
     	throw Error(format("Path '%1%' is not a valid component or state path") % store_or_statePath);
+    	*/
+
+	//TODO!!!!!!!!!!!!!!!!!!!! use revision !!!!!!!!!!!!!!!!!!!!! add timestamp ?????
+
+	PathSet referrers;
+    
+    if(isValidPathTxn(txn, store_or_statePath) || isValidStatePathTxn(txn, store_or_statePath)){
+
+		//Lookup its where it points to if its shared
+		Path store_or_statePath_ns;
+		if(isValidStatePathTxn(txn, store_or_statePath))
+    		store_or_statePath_ns = toNonSharedPathTxn(txn, store_or_statePath);
+    	
+		map<string, int> possible_referrers;	//referrer at timestamp
+
+		Strings keys;
+   		nixDB.enumTable(txn, table, keys);
+   		for (Strings::iterator i = keys.begin(); i != keys.end(); ++i){
+   			if( (*i).substr(0, store_or_statePath_ns.length()) == store_or_statePath_ns){
+   				  if( (*i).length() == store_or_statePath_ns.length() )
+   				  	referrers.insert(*i);									//referrers from components
+   				  else{
+   				  	
+   				  	//now filter the latest of each value
+   				  	Path empty;
+   				  	int timestamp;
+   				  	nixDB.splitDBKey(*i, empty, timestamp);
+   				  	
+   				  	string data;
+   				  	nixDB.queryString(txn, table, *i, data);
+   				  	
+   				  	Path statePath; 
+   				  	int added_or_removed;
+   				  	nixDB.splitDBKey(data, statePath, added_or_removed);
+   				  	
+   				  	if(possible_referrers[statePath] != 0)		//if not yet set
+   				  		if(timestamp > possible_referrers[statePath])
+   				  			possible_referrers[statePath] = timestamp;
+   				  	else
+   				  		possible_referrers[statePath] = timestamp;
+   				  }
+   			}
+   		}
+   		
+   		//Check wheter the latest was added or deleted
+   		for (map<string, int>::iterator i = possible_referrers.begin(); i != possible_referrers.end(); ++i){
+   			
+   			string data;
+   			nixDB.queryString(txn, table, nixDB.mergeToDBKey(store_or_statePath_ns, (*i).second), data);
+   			
+   			Path statePath; 
+   			int added_or_removed;
+   			nixDB.splitDBKey(data, statePath, added_or_removed);
+   			
+   			if(added_or_removed == 1)		//now filter on -0 (removed) or -1 (added)
+   				referrers.insert(statePath);
+   		}
+   		
+    	return referrers;
+    }
+    else
+    	throw Error(format("Path '%1%' is not a valid component or state path") % store_or_statePath);
+    
+    
 }
 
 static PathSet getReferrers(const Transaction & txn, const Path & store_or_statePath, const int revision)

src/nix-state/nix-state.cc

@@ -415,8 +415,11 @@ void scanAndUpdateAllReferencesTxn(const Transaction & txn, const Path & statePa
     		printMsg(lvlError, format("Removed state reference found!: '%1%' in state path '%2%'") % (*i) % statePath);
 
 	//If any changes are detected: register the paths valid with a new revision number
-   	if(diff_references_added.size() != 0 || diff_references_removed.size() != 0 ||
+   	//if(diff_references_added.size() != 0 || diff_references_removed.size() != 0 ||
-   	   diff_state_references_added.size() != 0 || diff_state_references_removed.size() != 0 )
+   	//diff_state_references_added.size() != 0 || diff_state_references_removed.size() != 0 )
+
+	//We always set the referernces so we know they were scanned (maybe the same) at a certain time
+	if(true)
    	{
 	   	printMsg(lvlError, format("Updating new references for statepath: '%1%'") % statePath);
 	   	Path drvPath = queryStatePathDrvTxn(txn, statePath);
@@ -638,10 +641,15 @@ void run(Strings args)
 	test["a"] = "a";
 	for (map<string, string>::const_iterator j = test.begin(); j != test.end(); ++j)
 		printMsg(lvlError, format("KEY: '%1%'") % (*j).first);
-	
 	printMsg(lvlError, format("NOW: '%1%'") % getTimeStamp());
 	return;
 
+	map<string, int> test;
+	test["a"] = 1;
+	test["b"] = 2;
+	printMsg(lvlError, format("NOW: '%1%'") % test["q"]);
+	return;	
+	
 	*/
 
 	/* test */