diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 48d418913..3203b26bf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1702,10 +1702,11 @@ void DerivationGoal::computeClosure() /* Get rid of all weird permissions. */ canonicalisePathMetaData(path); - /* For this output path, find the references to other paths contained in it. */ + /* For this output path, find the component references to other paths contained in it. */ PathSet references = scanForReferences(path, allPaths); + allReferences[path] = references; - //TODO comment + /* For this output path, find the state references to other paths contained in it. */ PathSet output_state_references = scanForReferences(path, allStatePaths); allStateReferences[path] = output_state_references; @@ -1720,8 +1721,7 @@ void DerivationGoal::computeClosure() else debug(format("referenced input: `%1%'") % *i); } - - allReferences[path] = references; + /* If the derivation specifies an `allowedReferences' attribute (containing a list of paths that the output may @@ -1734,6 +1734,8 @@ void DerivationGoal::computeClosure() throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); } + //TODO !!!!!!!!!!!!!!!!!!!!!!!! AllowedStateReferences?? + /* Hash the contents of the path. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. !!! inefficient: it would be nice @@ -1741,12 +1743,12 @@ void DerivationGoal::computeClosure() contentHashes[path] = hashPath(htSHA256, path); } - //Scan the state Path - /* For this state-output path, find the references to other paths contained in it. - * Get the state paths (instead of out paths) from all components, and then call - * scanForReferences(). - */ - //If state is enabled: Seaches for state and component references in the state path + /* We already scanned for [Component references in Component paths] //1 + * and [Component paths for state references] //2 + * + * If state is enabled for the path we: + * [scan for and state references and component references in the state path] //3,4 + */ if(isStateDrvTxn(noTxn, drv)){ //TODO Path statePath = drv.stateOutputs.find("state")->second.statepath; printMsg(lvlTalkative, format("scanning for component and state references inside `%1%'") % statePath); @@ -1772,14 +1774,6 @@ void DerivationGoal::computeClosure() for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) { - //printMsg(lvlError, format("SetValidPath: %1%") % i->second.path); - /* - registerValidPath(txn, i->second.path, - contentHashes[i->second.path], - allReferences[i->second.path], - PathSet(), //dummy stateReferences - drvPath);*/ - registerValidPath(txn, i->second.path, //component path contentHashes[i->second.path], @@ -1791,8 +1785,9 @@ void DerivationGoal::computeClosure() //Register the state path valid if(isStateDrvTxn(txn, drv)) { + //TODO ONLY CALL THIS FUNCTION ON A NON-SHARED STATE PATH!!!!!!!!!!! + Path statePath = drv.stateOutputs.find("state")->second.statepath; - registerValidPath(txn, statePath, Hash(), //emtpy hash @@ -1801,95 +1796,6 @@ void DerivationGoal::computeClosure() drvPath); } - /* - * We first register alls paths as valid, and only scan for component references. - * Now that those paths are registered as valid, we're able to call queryDeriversStatePath - * - * We already scanned for [Component references in Component paths] //1 - * Now we scan in [Component paths for state references] //2 - * - * If state is enabled for the path we: - * [scan for and state references and component references in the state path] //3,4 - */ - - - - //TODO REMOVE? - /* - PathSet allStatePaths2; - for (PathSet::const_iterator i = allPaths.begin(); i != allPaths.end(); i++){ - Path componentPath = *i; - - if(isStateComponentTxn(txn, componentPath)){ - //printMsg(lvlError, format("Scanning for state path: %1%") % componentPath); - - //TODO A HACK !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - //We should only get the state paths of the derivers in the build closure from drv. - //The we can dismiss the call to queryDeriversStatePath(...), and we only have to do registerValidPath once - - PathSet stateRefs = queryDeriversStatePath(txn, componentPath ,"*",getCallingUserName()); - allStatePaths2 = mergePathSets(stateRefs, allStatePaths2); - } - } - - //Scan all output paths ... comment TODO - for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) - { - Path path = i->second.path; - - //We scan for state references in the component path - PathSet output_state_references = scanForReferences(path, allStatePaths2); - - //debugging - for (PathSet::const_iterator i = allStatePaths2.begin(); i != allStatePaths2.end(); i++) - debug(format("all possible StatePaths: %1%") % (*i)); - for (PathSet::const_iterator i = output_state_references.begin(); i != output_state_references.end(); i++) - debug(format("state References scanned: %1%") % (*i)); - - allStateReferences[path] = output_state_references; - } - - //Scan the state Path - /* For this state-output path, find the references to other paths contained in it. - * Get the state paths (instead of out paths) from all components, and then call - * scanForReferences(). - * / - //If state is enabled: Seaches for state and component references in the state path - if(isStateDrvTxn(txn, drv)){ - Path statePath = drv.stateOutputs.find("state")->second.statepath; - printMsg(lvlTalkative, format("scanning for component and state references inside `%1%'") % statePath); - - state_references = scanForReferences(statePath, allPaths); - state_stateReferences = scanForReferences(statePath, allStatePaths2); - } - - //Register all outputs now valid - for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) - { - registerValidPath(txn, - i->second.path, //component path - contentHashes[i->second.path], - allReferences[i->second.path], //set of component-references - allStateReferences[i->second.path], //set of state-references - drvPath); - } - - //Register the state path valid - if(isStateDrvTxn(txn, drv)) - { - Path statePath = drv.stateOutputs.find("state")->second.statepath; - - registerValidPath(txn, - statePath, - Hash(), //emtpy hash - state_references, - state_stateReferences, - drvPath); - } - - */ - txn.commit(); /* It is now safe to delete the lock files, since all future diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5a179ba71..d295d4aeb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1495,10 +1495,13 @@ PathSet mergeNewDerivationIntoList(const Path & storepath, const Path & newdrv, - Binary deployment (when called on an output path). - Source/binary deployment (when called on a derivation with `includeOutputs' set to true). + + + TODO Change comment, this can also take state paths */ -void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) +void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) { - computeFSClosure(storePath, paths, withComponents, withState); + computeFSClosure(storeOrstatePath, paths, withComponents, withState); if (includeOutputs) { for (PathSet::iterator i = paths.begin(); @@ -1513,9 +1516,9 @@ void storePathRequisites(const Path & storePath, const bool includeOutputs, Path } } -void LocalStore::storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) +void LocalStore::storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) { - return nix::storePathRequisites(storePath, includeOutputs, paths, withComponents, withState); + return nix::storePathRequisites(storeOrstatePath, includeOutputs, paths, withComponents, withState); } /* @@ -1568,69 +1571,142 @@ void getDependenciesAtBuildTime(const Transaction & txn, const Path & drvPath) } */ -void scanForAllReferences(const Transaction & txn, const Path & statePath) +//TODO include this call in the validate function +//TODO ONLY CALL THIS FUNCTION ON A NON-SHARED STATE PATH!!!!!!!!!!! +void scanAndUpdateAllReferencesTxn(const Transaction & txn, const Path & statePath + , PathSet & newFoundComponentReferences, PathSet & newFoundStateReferences) //only for recursion { - //get all possible state and component references + //Check if is a state Path + if(! isStateComponentTxn(txn, statePath)) + throw Error(format("This path '%1%' is not a state path") % statePath); + //TODO check if path is not a shared path ! + //TODO + + //get all possible state and component references Paths referencesKeys; Paths referencesKeys2; Paths stateReferencesKeys; nixDB.enumTable(txn, dbReferences, referencesKeys); nixDB.enumTable(txn, dbStateReferences, stateReferencesKeys); - for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i){ - string s = *i; - if (s.substr(0,10) == "/nix/state") - printMsg(lvlError, format("referencesKeys: %1%") % s); - } - for (Paths::iterator i = stateReferencesKeys.begin(); i != stateReferencesKeys.end(); ++i){ - string s = *i; - if (s.substr(0,10) == "/nix/state") - printMsg(lvlError, format("stateReferencesKeys: %1%") % s); - } + for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i) + debug(format("referencesKeys: %1%") % *i); + for (Paths::iterator i = stateReferencesKeys.begin(); i != stateReferencesKeys.end(); ++i) + debug(format("stateReferencesKeys: %1%") % *i); - //Remove derivation paths ..... + //Remove derivation paths for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i){ string path = *i; - //printMsg(lvlError, format("refkey: %1%") % path); if(path.substr(path.length() - 4,path.length()) != ".drv") //TODO HACK: we should have a typed table or a seperate table .... referencesKeys2.push_back(path); } - //Merge - PathSet scanPaths = mergePathSets(PathSet(referencesKeys2.begin(), referencesKeys2.end()), - PathSet(stateReferencesKeys.begin(), stateReferencesKeys.end())); + //Scan in for component and state references + PathSet state_references = scanForReferences(statePath, PathSet(referencesKeys2.begin(), referencesKeys2.end())); + PathSet state_stateReferences = scanForReferences(statePath, PathSet(stateReferencesKeys.begin(), stateReferencesKeys.end())); - //for (PathSet::iterator i = scanPaths.begin(); i != scanPaths.end(); ++i) - // printMsg(lvlError, format("SCANNED: %1%") % *i); + //Retrieve old references + PathSet old_references; + PathSet old_state_references; + queryReferences(txn, statePath, old_references); + queryStateReferences(txn, statePath, old_state_references); - //Scan in statePath - PathSet scannedReferences = scanForReferences(statePath, scanPaths); - - for (PathSet::iterator i = scannedReferences.begin(); i != scannedReferences.end(); ++i) - printMsg(lvlError, format("RESULT: %1%") % *i); + //Check for added and removed paths + PathSet diff_references_removed; + PathSet diff_references_added; + pathSets_difference(state_references, old_references, diff_references_removed, diff_references_added); + PathSet diff_state_references_removed; + PathSet diff_state_references_added; + pathSets_difference(state_stateReferences, old_state_references, diff_state_references_removed, diff_state_references_added); + + newFoundComponentReferences = diff_references_added; + newFoundStateReferences = diff_state_references_added; + + //Print error, but we could also throw an error. + if(diff_references_removed.size() != 0) + for (PathSet::iterator i = diff_references_removed.begin(); i != diff_references_removed.end(); ++i) + printMsg(lvlError, format("Removed component reference found!: '%1%' in state path '%2%'") % (*i) % statePath); + if(diff_references_added.size() != 0) + for (PathSet::iterator i = diff_references_added.begin(); i != diff_references_added.end(); ++i) + printMsg(lvlError, format("Added component reference found!: '%1%' in state path '%2%'") % (*i) % statePath); + if(diff_state_references_removed.size() != 0) + for (PathSet::iterator i = diff_state_references_removed.begin(); i != diff_state_references_removed.end(); ++i) + printMsg(lvlError, format("Removed state reference found!: '%1%' in state path '%2%'") % (*i) % statePath); + if(diff_state_references_added.size() != 0) + for (PathSet::iterator i = diff_state_references_added.begin(); i != diff_state_references_added.end(); ++i) + printMsg(lvlError, format("Added state reference found!: '%1%' in state path '%2%'") % (*i) % statePath); + + //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /* + Register Valid again if neccesary + update the extra references in a new table??? why??? + (remember we need to keep the old as the basis, and things can change, the db is not consistent anymore then ....) + But we also dont want useless refereces ...... + + TODO: solution, + ALSO: + + Update the 2 references tables: + all state paths get: run number + state references + revision numbers) THIS ONE !!!!!! + + A1(STATEPATH) --> UPDATE ALL STATE REFERENCES IN DB (TRANSACTION) + A2(STATEPATH [+ REV]) --> GIVE ALL STATE REFERENCES IN DB + + B1(STATEPATH) --> SCAN ALL REFERENCES IN DB, LINK TO REVISION NUMBERS (WITH CALL TO A2) (TRANSACTION) + B2(STATEPATH [+ REV]) --> GIVES ALL REFERENCES + + + + //update all revision numbers (transaction) NEW FUN THAT UPDATES THE REVISIONS + //update all revision numbers + references (transaction) + //update all references + */ + + /* + if(diff_references_added.size() != 0 || diff_state_references_added.size() != 0){ + printMsg(lvlError, format("Updating new references for statepath: '%1%'")% statePath); + Path drvPath = queryStatePathDrv(txn, statePath); + registerValidPath(txn, + statePath, + Hash(), //emtpy hash + state_references, + state_stateReferences, + drvPath); + } + */ } -void LocalStore::scanForAllReferences(const Path & statePath) +void LocalStore::scanAndUpdateAllReferences(const Path & statePath) { - return nix::scanForAllReferences(noTxn, statePath); + PathSet empty; + return nix::scanAndUpdateAllReferencesTxn(noTxn, statePath, empty, empty); } -void scanForAllReferencesRecusively(const Transaction & txn, const Path & storePath) +void scanAndUpdateAllReferencesRecusively(const Transaction & txn, const Path & storeOrStatePath) //TODO Can also work for statePaths??? { - //get all state references + //get all state references recursively + PathSet statePaths; + storePathRequisites(storeOrStatePath, false, statePaths, false, true); //TODO CAN THIS ??? - //call scanForAllReferences on all - - //compare results with the current registered component and state paths - - //update the extra references in a new table??? why??? - //(remember we need to keep the old as the basis, and things can change, the db is not consisten anymore then ....) and error if neseccary + //call scanForAllReferences again on all newly found statePaths + for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) + { + //... and merge + PathSet newFoundComponentReferences; + PathSet newFoundStateReferences; + scanAndUpdateAllReferencesTxn(txn, *i, newFoundComponentReferences, newFoundStateReferences); + PathSet allNewReferences = pathSets_union(newFoundComponentReferences, newFoundStateReferences); + + //Call the function recursively again on all newly found references //TODO test if this doesnt go into an infinite loop + for (PathSet::iterator j = allNewReferences.begin(); j != allNewReferences.end(); ++j) + scanAndUpdateAllReferencesRecusively(txn, *j); + } } -void LocalStore::scanForAllReferencesRecusively(const Path & storePath) +void LocalStore::scanAndUpdateAllReferencesRecusively(const Path & storeOrStatePath) { - return nix::scanForAllReferencesRecusively(noTxn, storePath); + return nix::scanAndUpdateAllReferencesRecusively(noTxn, storeOrStatePath); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 279ddeace..077547ca4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -96,11 +96,11 @@ public: bool isStateDrv(const Derivation & drv); - void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState); + void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState); - void scanForAllReferences(const Path & statePath); + void scanAndUpdateAllReferences(const Path & statePath); - void scanForAllReferencesRecusively(const Path & storePath); + void scanAndUpdateAllReferencesRecusively(const Path & storeOrstatePath); }; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index a7f2db25b..adf0bbd15 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -67,7 +67,7 @@ void computeFSClosureRec(const Path & path, PathSet & paths, const bool & flipDi } PathSet allReferences; - allReferences = mergePathSets(references, stateReferences); + allReferences = pathSets_union(references, stateReferences); for (PathSet::iterator i = allReferences.begin(); i != allReferences.end(); ++i) computeFSClosureRec(*i, paths, flipDirection); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7a2ec505b..a657eba41 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -452,19 +452,19 @@ bool RemoteStore::isStateDrv(const Derivation & drv) } //TODO -void RemoteStore::storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) +void RemoteStore::storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) { } //TODO -void RemoteStore::scanForAllReferences(const Path & statePath) +void RemoteStore::scanAndUpdateAllReferences(const Path & statePath) { } //TODO -void RemoteStore::scanForAllReferencesRecusively(const Path & storePath) +void RemoteStore::scanAndUpdateAllReferencesRecusively(const Path & storeOrstatePath) { } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 6de3376f9..5cdd9f1d0 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -84,11 +84,11 @@ public: bool isStateDrv(const Derivation & drv); - void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState); + void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState); - void scanForAllReferences(const Path & statePath); + void scanAndUpdateAllReferences(const Path & statePath); - void scanForAllReferencesRecusively(const Path & storePath); + void scanAndUpdateAllReferencesRecusively(const Path & storeOrstatePath); private: AutoCloseFD fdSocket; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 97312d22d..e0b3f5b1b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -215,13 +215,13 @@ public: virtual bool isStateDrv(const Derivation & drv) = 0; /* TODO */ - virtual void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) = 0; + virtual void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState) = 0; /* TODO */ - virtual void scanForAllReferences(const Path & statePath) = 0; + virtual void scanAndUpdateAllReferences(const Path & statePath) = 0; /* TODO */ - virtual void scanForAllReferencesRecusively(const Path & storePath) = 0; + virtual void scanAndUpdateAllReferencesRecusively(const Path & storeOrstatePath) = 0; }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index db1fc0fc4..4387c91f0 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1146,28 +1146,78 @@ string getCallingUserName() return username; } -//merges two PathSets into one, removing doubles -PathSet mergePathSets(const PathSet & paths1, const PathSet & paths2) +//merges two PathSets into one, removing doubles (union) +PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2) { - PathSet merged = paths2; - - for (PathSet::iterator i = paths1.begin(); i != paths1.end(); ++i) //were inserting all paths from pathset1 into pathset2 - { - bool alreadyExists = false; - for (PathSet::iterator j = paths2.begin(); j != paths2.end(); ++j) //search in p2 for duplicates - { - if(*i == *j){ - alreadyExists = true; - break; - } - } - - if( !alreadyExists ){ //insert into p2 if not duplicate - merged.insert(*i); - } - } + /* + for (PathSet::iterator i = paths1.begin(); i != paths1.end(); ++i) + printMsg(lvlError, format("paths1: '%1%'") % (*i)); + for (PathSet::iterator i = paths2.begin(); i != paths2.end(); ++i) + printMsg(lvlError, format("paths2: '%1%'") % (*i)); + */ - return merged; + vector vector1(paths1.begin(), paths1.end()); + vector vector2(paths2.begin(), paths2.end()); + vector setResult; + + set_union(vector1.begin(), vector1.end(),vector2.begin(), vector2.end(), back_inserter(setResult)); + + //Also available: + //set_symmetric_difference + //set_intersection + + PathSet diff; + for(int i=0; iscanForAllReferences("/nix/state/0qhlpz1ji4gvg3j6nk5vkcddmi3m5x1r-hellohardcodedstateworld-1.0-test2"); + store->scanForAllReferences("/nix/state/i06flm2ahq5s0x3633z30dnav9f1wkb5-hellohardcodedstateworld-dep1-1.0-test"); return; + */ /* test */