1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-26 04:00:59 +01:00

added scanAndUpdateAllReferencesTxn(..) moving on to create a db-table that can save state-revision-closures and state-revision-reference-closures

This commit is contained in:
Wouter den Breejen 2007-07-02 19:15:10 +00:00
parent 1c3ec86c39
commit ad2b815b5e
10 changed files with 235 additions and 184 deletions

View file

@ -1702,10 +1702,11 @@ void DerivationGoal::computeClosure()
/* Get rid of all weird permissions. */
canonicalisePathMetaData(path);
/* For this output path, find the references to other paths contained in it. */
/* For this output path, find the component references to other paths contained in it. */
PathSet references = scanForReferences(path, allPaths);
allReferences[path] = references;
//TODO comment
/* For this output path, find the state references to other paths contained in it. */
PathSet output_state_references = scanForReferences(path, allStatePaths);
allStateReferences[path] = output_state_references;
@ -1721,7 +1722,6 @@ void DerivationGoal::computeClosure()
debug(format("referenced input: `%1%'") % *i);
}
allReferences[path] = references;
/* If the derivation specifies an `allowedReferences'
attribute (containing a list of paths that the output may
@ -1734,6 +1734,8 @@ void DerivationGoal::computeClosure()
throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
}
//TODO !!!!!!!!!!!!!!!!!!!!!!!! AllowedStateReferences??
/* Hash the contents of the path. The hash is stored in the
database so that we can verify later on whether nobody has
messed with the store. !!! inefficient: it would be nice
@ -1741,12 +1743,12 @@ void DerivationGoal::computeClosure()
contentHashes[path] = hashPath(htSHA256, path);
}
//Scan the state Path
/* For this state-output path, find the references to other paths contained in it.
* Get the state paths (instead of out paths) from all components, and then call
* scanForReferences().
/* We already scanned for [Component references in Component paths] //1
* and [Component paths for state references] //2
*
* If state is enabled for the path we:
* [scan for and state references and component references in the state path] //3,4
*/
//If state is enabled: Seaches for state and component references in the state path
if(isStateDrvTxn(noTxn, drv)){ //TODO
Path statePath = drv.stateOutputs.find("state")->second.statepath;
printMsg(lvlTalkative, format("scanning for component and state references inside `%1%'") % statePath);
@ -1772,14 +1774,6 @@ void DerivationGoal::computeClosure()
for (DerivationOutputs::iterator i = drv.outputs.begin();
i != drv.outputs.end(); ++i)
{
//printMsg(lvlError, format("SetValidPath: %1%") % i->second.path);
/*
registerValidPath(txn, i->second.path,
contentHashes[i->second.path],
allReferences[i->second.path],
PathSet(), //dummy stateReferences
drvPath);*/
registerValidPath(txn,
i->second.path, //component path
contentHashes[i->second.path],
@ -1791,8 +1785,9 @@ void DerivationGoal::computeClosure()
//Register the state path valid
if(isStateDrvTxn(txn, drv))
{
Path statePath = drv.stateOutputs.find("state")->second.statepath;
//TODO ONLY CALL THIS FUNCTION ON A NON-SHARED STATE PATH!!!!!!!!!!!
Path statePath = drv.stateOutputs.find("state")->second.statepath;
registerValidPath(txn,
statePath,
Hash(), //emtpy hash
@ -1801,95 +1796,6 @@ void DerivationGoal::computeClosure()
drvPath);
}
/*
* We first register alls paths as valid, and only scan for component references.
* Now that those paths are registered as valid, we're able to call queryDeriversStatePath
*
* We already scanned for [Component references in Component paths] //1
* Now we scan in [Component paths for state references] //2
*
* If state is enabled for the path we:
* [scan for and state references and component references in the state path] //3,4
*/
//TODO REMOVE?
/*
PathSet allStatePaths2;
for (PathSet::const_iterator i = allPaths.begin(); i != allPaths.end(); i++){
Path componentPath = *i;
if(isStateComponentTxn(txn, componentPath)){
//printMsg(lvlError, format("Scanning for state path: %1%") % componentPath);
//TODO A HACK !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//We should only get the state paths of the derivers in the build closure from drv.
//The we can dismiss the call to queryDeriversStatePath(...), and we only have to do registerValidPath once
PathSet stateRefs = queryDeriversStatePath(txn, componentPath ,"*",getCallingUserName());
allStatePaths2 = mergePathSets(stateRefs, allStatePaths2);
}
}
//Scan all output paths ... comment TODO
for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)
{
Path path = i->second.path;
//We scan for state references in the component path
PathSet output_state_references = scanForReferences(path, allStatePaths2);
//debugging
for (PathSet::const_iterator i = allStatePaths2.begin(); i != allStatePaths2.end(); i++)
debug(format("all possible StatePaths: %1%") % (*i));
for (PathSet::const_iterator i = output_state_references.begin(); i != output_state_references.end(); i++)
debug(format("state References scanned: %1%") % (*i));
allStateReferences[path] = output_state_references;
}
//Scan the state Path
/* For this state-output path, find the references to other paths contained in it.
* Get the state paths (instead of out paths) from all components, and then call
* scanForReferences().
* /
//If state is enabled: Seaches for state and component references in the state path
if(isStateDrvTxn(txn, drv)){
Path statePath = drv.stateOutputs.find("state")->second.statepath;
printMsg(lvlTalkative, format("scanning for component and state references inside `%1%'") % statePath);
state_references = scanForReferences(statePath, allPaths);
state_stateReferences = scanForReferences(statePath, allStatePaths2);
}
//Register all outputs now valid
for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)
{
registerValidPath(txn,
i->second.path, //component path
contentHashes[i->second.path],
allReferences[i->second.path], //set of component-references
allStateReferences[i->second.path], //set of state-references
drvPath);
}
//Register the state path valid
if(isStateDrvTxn(txn, drv))
{
Path statePath = drv.stateOutputs.find("state")->second.statepath;
registerValidPath(txn,
statePath,
Hash(), //emtpy hash
state_references,
state_stateReferences,
drvPath);
}
*/
txn.commit();
/* It is now safe to delete the lock files, since all future

View file

@ -1495,10 +1495,13 @@ PathSet mergeNewDerivationIntoList(const Path & storepath, const Path & newdrv,
- Binary deployment (when called on an output path).
- Source/binary deployment (when called on a derivation with
`includeOutputs' set to true).
TODO Change comment, this can also take state paths
*/
void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState)
void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState)
{
computeFSClosure(storePath, paths, withComponents, withState);
computeFSClosure(storeOrstatePath, paths, withComponents, withState);
if (includeOutputs) {
for (PathSet::iterator i = paths.begin();
@ -1513,9 +1516,9 @@ void storePathRequisites(const Path & storePath, const bool includeOutputs, Path
}
}
void LocalStore::storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState)
void LocalStore::storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState)
{
return nix::storePathRequisites(storePath, includeOutputs, paths, withComponents, withState);
return nix::storePathRequisites(storeOrstatePath, includeOutputs, paths, withComponents, withState);
}
/*
@ -1568,69 +1571,142 @@ void getDependenciesAtBuildTime(const Transaction & txn, const Path & drvPath)
}
*/
void scanForAllReferences(const Transaction & txn, const Path & statePath)
//TODO include this call in the validate function
//TODO ONLY CALL THIS FUNCTION ON A NON-SHARED STATE PATH!!!!!!!!!!!
void scanAndUpdateAllReferencesTxn(const Transaction & txn, const Path & statePath
, PathSet & newFoundComponentReferences, PathSet & newFoundStateReferences) //only for recursion
{
//get all possible state and component references
//Check if is a state Path
if(! isStateComponentTxn(txn, statePath))
throw Error(format("This path '%1%' is not a state path") % statePath);
//TODO check if path is not a shared path !
//TODO
//get all possible state and component references
Paths referencesKeys;
Paths referencesKeys2;
Paths stateReferencesKeys;
nixDB.enumTable(txn, dbReferences, referencesKeys);
nixDB.enumTable(txn, dbStateReferences, stateReferencesKeys);
for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i){
string s = *i;
if (s.substr(0,10) == "/nix/state")
printMsg(lvlError, format("referencesKeys: %1%") % s);
}
for (Paths::iterator i = stateReferencesKeys.begin(); i != stateReferencesKeys.end(); ++i){
string s = *i;
if (s.substr(0,10) == "/nix/state")
printMsg(lvlError, format("stateReferencesKeys: %1%") % s);
}
for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i)
debug(format("referencesKeys: %1%") % *i);
for (Paths::iterator i = stateReferencesKeys.begin(); i != stateReferencesKeys.end(); ++i)
debug(format("stateReferencesKeys: %1%") % *i);
//Remove derivation paths .....
//Remove derivation paths
for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i){
string path = *i;
//printMsg(lvlError, format("refkey: %1%") % path);
if(path.substr(path.length() - 4,path.length()) != ".drv") //TODO HACK: we should have a typed table or a seperate table ....
referencesKeys2.push_back(path);
}
//Merge
PathSet scanPaths = mergePathSets(PathSet(referencesKeys2.begin(), referencesKeys2.end()),
PathSet(stateReferencesKeys.begin(), stateReferencesKeys.end()));
//Scan in for component and state references
PathSet state_references = scanForReferences(statePath, PathSet(referencesKeys2.begin(), referencesKeys2.end()));
PathSet state_stateReferences = scanForReferences(statePath, PathSet(stateReferencesKeys.begin(), stateReferencesKeys.end()));
//for (PathSet::iterator i = scanPaths.begin(); i != scanPaths.end(); ++i)
// printMsg(lvlError, format("SCANNED: %1%") % *i);
//Retrieve old references
PathSet old_references;
PathSet old_state_references;
queryReferences(txn, statePath, old_references);
queryStateReferences(txn, statePath, old_state_references);
//Scan in statePath
PathSet scannedReferences = scanForReferences(statePath, scanPaths);
//Check for added and removed paths
PathSet diff_references_removed;
PathSet diff_references_added;
pathSets_difference(state_references, old_references, diff_references_removed, diff_references_added);
PathSet diff_state_references_removed;
PathSet diff_state_references_added;
pathSets_difference(state_stateReferences, old_state_references, diff_state_references_removed, diff_state_references_added);
for (PathSet::iterator i = scannedReferences.begin(); i != scannedReferences.end(); ++i)
printMsg(lvlError, format("RESULT: %1%") % *i);
newFoundComponentReferences = diff_references_added;
newFoundStateReferences = diff_state_references_added;
//Print error, but we could also throw an error.
if(diff_references_removed.size() != 0)
for (PathSet::iterator i = diff_references_removed.begin(); i != diff_references_removed.end(); ++i)
printMsg(lvlError, format("Removed component reference found!: '%1%' in state path '%2%'") % (*i) % statePath);
if(diff_references_added.size() != 0)
for (PathSet::iterator i = diff_references_added.begin(); i != diff_references_added.end(); ++i)
printMsg(lvlError, format("Added component reference found!: '%1%' in state path '%2%'") % (*i) % statePath);
if(diff_state_references_removed.size() != 0)
for (PathSet::iterator i = diff_state_references_removed.begin(); i != diff_state_references_removed.end(); ++i)
printMsg(lvlError, format("Removed state reference found!: '%1%' in state path '%2%'") % (*i) % statePath);
if(diff_state_references_added.size() != 0)
for (PathSet::iterator i = diff_state_references_added.begin(); i != diff_state_references_added.end(); ++i)
printMsg(lvlError, format("Added state reference found!: '%1%' in state path '%2%'") % (*i) % statePath);
//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
/*
Register Valid again if neccesary
update the extra references in a new table??? why???
(remember we need to keep the old as the basis, and things can change, the db is not consistent anymore then ....)
But we also dont want useless refereces ......
TODO: solution,
ALSO:
Update the 2 references tables:
all state paths get: run number + state references + revision numbers) THIS ONE !!!!!!
A1(STATEPATH) --> UPDATE ALL STATE REFERENCES IN DB (TRANSACTION)
A2(STATEPATH [+ REV]) --> GIVE ALL STATE REFERENCES IN DB
B1(STATEPATH) --> SCAN ALL REFERENCES IN DB, LINK TO REVISION NUMBERS (WITH CALL TO A2) (TRANSACTION)
B2(STATEPATH [+ REV]) --> GIVES ALL REFERENCES
//update all revision numbers (transaction) NEW FUN THAT UPDATES THE REVISIONS
//update all revision numbers + references (transaction)
//update all references
*/
/*
if(diff_references_added.size() != 0 || diff_state_references_added.size() != 0){
printMsg(lvlError, format("Updating new references for statepath: '%1%'")% statePath);
Path drvPath = queryStatePathDrv(txn, statePath);
registerValidPath(txn,
statePath,
Hash(), //emtpy hash
state_references,
state_stateReferences,
drvPath);
}
*/
}
void LocalStore::scanForAllReferences(const Path & statePath)
void LocalStore::scanAndUpdateAllReferences(const Path & statePath)
{
return nix::scanForAllReferences(noTxn, statePath);
PathSet empty;
return nix::scanAndUpdateAllReferencesTxn(noTxn, statePath, empty, empty);
}
void scanForAllReferencesRecusively(const Transaction & txn, const Path & storePath)
void scanAndUpdateAllReferencesRecusively(const Transaction & txn, const Path & storeOrStatePath) //TODO Can also work for statePaths???
{
//get all state references
//get all state references recursively
PathSet statePaths;
storePathRequisites(storeOrStatePath, false, statePaths, false, true); //TODO CAN THIS ???
//call scanForAllReferences on all
//call scanForAllReferences again on all newly found statePaths
for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i)
{
//... and merge
PathSet newFoundComponentReferences;
PathSet newFoundStateReferences;
scanAndUpdateAllReferencesTxn(txn, *i, newFoundComponentReferences, newFoundStateReferences);
PathSet allNewReferences = pathSets_union(newFoundComponentReferences, newFoundStateReferences);
//compare results with the current registered component and state paths
//update the extra references in a new table??? why???
//(remember we need to keep the old as the basis, and things can change, the db is not consisten anymore then ....) and error if neseccary
//Call the function recursively again on all newly found references //TODO test if this doesnt go into an infinite loop
for (PathSet::iterator j = allNewReferences.begin(); j != allNewReferences.end(); ++j)
scanAndUpdateAllReferencesRecusively(txn, *j);
}
}
void LocalStore::scanForAllReferencesRecusively(const Path & storePath)
void LocalStore::scanAndUpdateAllReferencesRecusively(const Path & storeOrStatePath)
{
return nix::scanForAllReferencesRecusively(noTxn, storePath);
return nix::scanAndUpdateAllReferencesRecusively(noTxn, storeOrStatePath);
}

View file

@ -96,11 +96,11 @@ public:
bool isStateDrv(const Derivation & drv);
void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState);
void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState);
void scanForAllReferences(const Path & statePath);
void scanAndUpdateAllReferences(const Path & statePath);
void scanForAllReferencesRecusively(const Path & storePath);
void scanAndUpdateAllReferencesRecusively(const Path & storeOrstatePath);
};

View file

@ -67,7 +67,7 @@ void computeFSClosureRec(const Path & path, PathSet & paths, const bool & flipDi
}
PathSet allReferences;
allReferences = mergePathSets(references, stateReferences);
allReferences = pathSets_union(references, stateReferences);
for (PathSet::iterator i = allReferences.begin(); i != allReferences.end(); ++i)
computeFSClosureRec(*i, paths, flipDirection);

View file

@ -452,19 +452,19 @@ bool RemoteStore::isStateDrv(const Derivation & drv)
}
//TODO
void RemoteStore::storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState)
void RemoteStore::storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState)
{
}
//TODO
void RemoteStore::scanForAllReferences(const Path & statePath)
void RemoteStore::scanAndUpdateAllReferences(const Path & statePath)
{
}
//TODO
void RemoteStore::scanForAllReferencesRecusively(const Path & storePath)
void RemoteStore::scanAndUpdateAllReferencesRecusively(const Path & storeOrstatePath)
{
}

View file

@ -84,11 +84,11 @@ public:
bool isStateDrv(const Derivation & drv);
void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState);
void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState);
void scanForAllReferences(const Path & statePath);
void scanAndUpdateAllReferences(const Path & statePath);
void scanForAllReferencesRecusively(const Path & storePath);
void scanAndUpdateAllReferencesRecusively(const Path & storeOrstatePath);
private:
AutoCloseFD fdSocket;

View file

@ -215,13 +215,13 @@ public:
virtual bool isStateDrv(const Derivation & drv) = 0;
/* TODO */
virtual void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) = 0;
virtual void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) = 0;
/* TODO */
virtual void scanForAllReferences(const Path & statePath) = 0;
virtual void scanAndUpdateAllReferences(const Path & statePath) = 0;
/* TODO */
virtual void scanForAllReferencesRecusively(const Path & storePath) = 0;
virtual void scanAndUpdateAllReferencesRecusively(const Path & storeOrstatePath) = 0;
};

View file

@ -1146,28 +1146,78 @@ string getCallingUserName()
return username;
}
//merges two PathSets into one, removing doubles
PathSet mergePathSets(const PathSet & paths1, const PathSet & paths2)
//merges two PathSets into one, removing doubles (union)
PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2)
{
PathSet merged = paths2;
/*
for (PathSet::iterator i = paths1.begin(); i != paths1.end(); ++i)
printMsg(lvlError, format("paths1: '%1%'") % (*i));
for (PathSet::iterator i = paths2.begin(); i != paths2.end(); ++i)
printMsg(lvlError, format("paths2: '%1%'") % (*i));
*/
for (PathSet::iterator i = paths1.begin(); i != paths1.end(); ++i) //were inserting all paths from pathset1 into pathset2
{
bool alreadyExists = false;
for (PathSet::iterator j = paths2.begin(); j != paths2.end(); ++j) //search in p2 for duplicates
{
vector<Path> vector1(paths1.begin(), paths1.end());
vector<Path> vector2(paths2.begin(), paths2.end());
vector<Path> setResult;
set_union(vector1.begin(), vector1.end(),vector2.begin(), vector2.end(), back_inserter(setResult));
//Also available:
//set_symmetric_difference
//set_intersection
PathSet diff;
for(int i=0; i<setResult.size(); i++)
diff.insert(setResult[i]);
/*
for (PathSet::iterator i = diff.begin(); i != diff.end(); ++i)
printMsg(lvlError, format("diff: '%1%'") % (*i));
*/
return diff;
}
void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths)
{
/*
for (PathSet::iterator i = oldpaths.begin(); i != oldpaths.end(); ++i)
printMsg(lvlError, format("oldpaths: '%1%'") % (*i));
for (PathSet::iterator i = newpaths.begin(); i != newpaths.end(); ++i)
printMsg(lvlError, format("newpaths: '%1%'") % (*i));
*/
for (PathSet::iterator i = oldpaths.begin(); i != oldpaths.end(); ++i){
bool exists = false;
for (PathSet::iterator j = newpaths.begin(); j != newpaths.end(); ++j){
if(*i == *j){
alreadyExists = true;
exists = true;
break;
}
}
if( !alreadyExists ){ //insert into p2 if not duplicate
merged.insert(*i);
}
if(!exists)
removedpaths.insert(*i);
}
return merged;
for (PathSet::iterator i = newpaths.begin(); i != newpaths.end(); ++i){
bool exists = false;
for (PathSet::iterator j = oldpaths.begin(); j != oldpaths.end(); ++j){
if(*i == *j){
exists = true;
break;
}
}
if(!exists)
addedpaths.insert(*i);
}
/*
for (PathSet::iterator i = addedpaths.begin(); i != addedpaths.end(); ++i)
printMsg(lvlError, format("addedpaths: '%1%'") % (*i));
for (PathSet::iterator i = removedpaths.begin(); i != removedpaths.end(); ++i)
printMsg(lvlError, format("removedpaths: '%1%'") % (*i));
*/
}
}

View file

@ -308,7 +308,10 @@ bool IsDirectory(const string FileName);
string getCallingUserName();
/* TODO */
PathSet mergePathSets(const PathSet & paths1, const PathSet & paths2);
PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2);
/* TODO */
void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths);
}

View file

@ -334,6 +334,7 @@ static void opRunComponent(Strings opFlags, Strings opArgs)
//TODO
//Scan again??
//scanAndUpdateAllReferencesRecusively ...
}
}
@ -384,11 +385,26 @@ void run(Strings args)
printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/store/hbxqq4d67j2y21xzp7yp01qjfkcjjbc7-hellohardcodedstateworld-1.0"));
printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/state/0qhlpz1ji4gvg3j6nk5vkcddmi3m5x1r-hellohardcodedstateworld-1.0-test2"));
*/
PathSet p1;
PathSet p2;
PathSet p3;
PathSet p4;
p1.insert("a");
p1.insert("c"); //old
p1.insert("b");
p2.insert("b");
p2.insert("a");
p2.insert("cc"); //new
p2.insert("x"); //new
pathSets_difference(p1,p2,p3,p4);
pathSets_union(p1,p2);
store = openStore();
store->scanForAllReferences("/nix/state/0qhlpz1ji4gvg3j6nk5vkcddmi3m5x1r-hellohardcodedstateworld-1.0-test2");
store->scanForAllReferences("/nix/state/i06flm2ahq5s0x3633z30dnav9f1wkb5-hellohardcodedstateworld-dep1-1.0-test");
return;
*/
/* test */