diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f4b2dffd9..c344e5581 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -760,7 +760,7 @@ static Expr prim_toFile(EvalState & state, const ATermVector & args) string contents = evalString(state, args[1], context); PathSet refs; - PathSet stateRefs; //TODO TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + PathSet stateRefs; //refs refers to the file references, there are no state references in this case. for (PathSet::iterator i = context.begin(); i != context.end(); ++i) { if (isDerivation(*i)) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0c3f4b25a..a9d2aa53c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1686,9 +1686,7 @@ void DerivationGoal::computeClosure() /* For this output path, find the references to other paths contained in it. */ PathSet references = scanForReferences(path, allPaths); - - - + /* For debugging, print out the referenced and unreferenced paths. */ for (PathSet::iterator i = inputPaths.begin(); @@ -1703,7 +1701,6 @@ void DerivationGoal::computeClosure() allReferences[path] = references; - /* If the derivation specifies an `allowedReferences' attribute (containing a list of paths that the output may refer to), check that all references are in that list. !!! @@ -1746,15 +1743,18 @@ void DerivationGoal::computeClosure() } /* - * TODO COMMENT - * - * + * We first register alls paths as valid, and only scan for component references. + * Now that those paths are registered as valid, we're able to call queryDeriversStatePath * + * We already scanned for Component references in Component paths + * Now we scan in Component paths for state references * + * If state is enabled for the path we: + * scan for and state references and component references in the state path */ - for (DerivationOutputs::iterator i = drv.outputs.begin(); - i != drv.outputs.end(); ++i) + //TODO we scan for each output, be then we do multiple scans inside for the state path ..... + for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) { Path path = i->second.path; @@ -1766,23 +1766,32 @@ void DerivationGoal::computeClosure() for (PathSet::const_iterator i = allPaths.begin(); i != allPaths.end(); i++){ Path componentPath = *i; - //printMsg(lvlError, format("COMP: %1%") % (*i)); if(isStateComponentTxn(txn, componentPath)){ - printMsg(lvlError, format("COMP-STATE: %1%") % (*i)); + //printMsg(lvlError, format("Scanning for state path: %1%") % (*i)); PathSet stateRefs = queryDeriversStatePath(txn, componentPath ,"*",getCallingUserName()); allStatePaths = mergePathSets(stateRefs, allStatePaths); } } - PathSet stateReferences = scanForStateReferences(path, allStatePaths); + //We scan for state references in the component path + PathSet all_state_references = scanForStateReferences(path, allStatePaths); + + //If state is enabled: Seaches for state and component references in the state path + if(isStateDrvTxn(txn, drv)){ + Path statePath = drv.stateOutputs.find("state")->second.statepath; + PathSet state_references = scanForReferences(statePath, allPaths); + PathSet state_stateReferences = scanForStateReferences(statePath, allStatePaths); + all_state_references = mergePathSets(all_state_references, mergePathSets(state_references, state_stateReferences)); + } + for (PathSet::const_iterator i = allStatePaths.begin(); i != allStatePaths.end(); i++){ debug(format("all possible StatePaths: %1%") % (*i)); } - for (PathSet::const_iterator i = stateReferences.begin(); i != stateReferences.end(); i++){ + for (PathSet::const_iterator i = all_state_references.begin(); i != all_state_references.end(); i++){ debug(format("state References scanned: %1%") % (*i)); } - allStateReferences[path] = stateReferences; + allStateReferences[path] = all_state_references; } for (DerivationOutputs::iterator i = drv.outputs.begin(); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c08c73216..25a50c25c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -21,14 +21,13 @@ Path writeDerivation(const Derivation & drv, const string & name) { PathSet references; references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); - for (DerivationInputs::const_iterator i = drv.inputDrvs.begin(); - i != drv.inputDrvs.end(); ++i) + for (DerivationInputs::const_iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i) references.insert(i->first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - //TODO TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + //We only need to hash over inputSrcs and inputDrvs (I think ...) PathSet stateReferences; string suffix = name + drvExtension; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ff08c683e..818f63a2f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -44,7 +44,7 @@ static TableId dbValidPaths = 0; paths. */ static TableId dbReferences = 0; -/* dbReferences :: Path -> [Path] +/* dbStateReferences :: Path -> [Path] This table lists the outgoing file system state references for each output path that has been built by a Nix derivation. These are @@ -60,6 +60,13 @@ static TableId dbStateReferences = 0; referrer. */ static TableId dbReferrers = 0; +/* dbStateReferrers :: Path -> Path + + This table is just the reverse mapping of dbStateReferences. This table + can have duplicate keys, each corresponding value denoting a single + referrer. */ +static TableId dbStateReferrers = 0; + /* dbSubstitutes :: Path -> [[Path]] Each pair $(p, subs)$ tells Nix that it can use any of the @@ -159,12 +166,13 @@ LocalStore::LocalStore(bool reserveSpace) } dbValidPaths = nixDB.openTable("validpaths"); dbReferences = nixDB.openTable("references"); - dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */ //TODO ADD STATE REFERERS? + dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */ dbSubstitutes = nixDB.openTable("substitutes"); dbDerivers = nixDB.openTable("derivers"); dbStateInfo = nixDB.openTable("stateinfo"); dbStateCounters = nixDB.openTable("statecounters"); dbStateReferences = nixDB.openTable("references_state"); + dbStateReferrers = nixDB.openTable("referrers_state", true); /* must be sorted */ int curSchema = 0; Path schemaFN = nixDBPath + "/schema"; @@ -344,6 +352,16 @@ static PathSet getReferrers(const Transaction & txn, const Path & storePath) return referrers; } +static PathSet getStateReferrers(const Transaction & txn, const Path & storePath) //TODO this is just a copy of getReferrers, maybe make the function more generic? +{ + PathSet referrers; + Strings keys; + nixDB.enumTable(txn, dbStateReferrers, keys, storePath + string(1, (char) 0)); + for (Strings::iterator i = keys.begin(); i != keys.end(); ++i) + referrers.insert(stripPrefix(storePath, *i)); + return referrers; +} + void setReferences(const Transaction & txn, const Path & storePath, const PathSet & references, const PathSet & stateReferences) { @@ -369,20 +387,25 @@ void setReferences(const Transaction & txn, const Path & storePath, nixDB.setStrings(txn, dbStateReferences, storePath, Paths(stateReferences.begin(), stateReferences.end())); - //TODO THESE 2 ALSO FOR STATEREFS - /* Update the referrers mappings of all new referenced paths. */ - for (PathSet::const_iterator i = references.begin(); - i != references.end(); ++i) + for (PathSet::const_iterator i = references.begin(); i != references.end(); ++i) if (oldReferences2.find(*i) == oldReferences2.end()) nixDB.setString(txn, dbReferrers, addPrefix(*i, storePath), ""); - /* Remove referrer mappings from paths that are no longer - references. */ - for (Paths::iterator i = oldReferences.begin(); - i != oldReferences.end(); ++i) + /* Update the state referrers mappings of all new referenced paths. */ + for (PathSet::const_iterator i = stateReferences.begin(); i != stateReferences.end(); ++i) + if (oldStateReferences2.find(*i) == oldStateReferences2.end()) + nixDB.setString(txn, dbStateReferrers, addPrefix(*i, storePath), ""); + + /* Remove referrer mappings from paths that are no longer references. */ + for (Paths::iterator i = oldReferences.begin(); i != oldReferences.end(); ++i) if (references.find(*i) == references.end()) nixDB.delPair(txn, dbReferrers, addPrefix(*i, storePath)); + + /* Remove referrer mappings from paths that are no longer state references. */ + for (Paths::iterator i = oldStateReferences.begin(); i != oldStateReferences.end(); ++i) + if (stateReferences.find(*i) == stateReferences.end()) + nixDB.delPair(txn, dbStateReferrers, addPrefix(*i, storePath)); } @@ -415,15 +438,11 @@ void queryStateReferences(const Transaction & txn, stateReferences.insert(stateReferences2.begin(), stateReferences2.end()); } -void LocalStore::queryStateReferences(const Path & storePath, - PathSet & stateReferences) +void LocalStore::queryStateReferences(const Path & storePath, PathSet & stateReferences) { nix::queryStateReferences(noTxn, storePath, stateReferences); } - -//TODO getStateReferrers.... - void queryReferrers(const Transaction & txn, const Path & storePath, PathSet & referrers) { @@ -441,6 +460,21 @@ void LocalStore::queryReferrers(const Path & storePath, } +void queryStateReferrers(const Transaction & txn, const Path & storePath, PathSet & stateReferrers) +{ + if (!isRealisablePath(txn, storePath)) + throw Error(format("path `%1%' is not valid") % storePath); + PathSet stateReferrers2 = getStateReferrers(txn, storePath); + stateReferrers.insert(stateReferrers2.begin(), stateReferrers2.end()); +} + + +void LocalStore::queryStateReferrers(const Path & storePath, PathSet & stateReferrers) +{ + nix::queryStateReferrers(noTxn, storePath, stateReferrers); +} + + void setDeriver(const Transaction & txn, const Path & storePath, const Path & deriver) { assertStorePath(storePath); @@ -498,19 +532,31 @@ bool LocalStore::isStateComponent(const Path & storePath) return nix::isStateComponentTxn(noTxn, storePath); } + //TODO Add and .. -bool isStateDrv(const Path & drvPath) +bool isStateDrvPathTxn(const Transaction & txn, const Path & drvPath) +{ + Derivation drv = derivationFromPath(drvPath); + return isStateDrvTxn(txn, drv); +} + +bool LocalStore::isStateDrvPath(const Path & isStateDrv) +{ + return nix::isStateDrvPathTxn(noTxn, isStateDrv); +} + +//TODO Add and .. +bool isStateDrvTxn(const Transaction & txn, Derivation drv) { - Derivation drv = derivationFromPath(drvPath); //maybe redirect the out path to isStateComponent? if (drv.stateOutputs.size() != 0) return true; else return false; } -bool LocalStore::isStateDrv(const Path & isStateDrv) +bool LocalStore::isStateDrv(Derivation drv) { - return nix::isStateDrv(isStateDrv); + return nix::isStateDrvTxn(noTxn, drv); } Path queryDeriver(const Transaction & txn, const Path & storePath) @@ -1088,6 +1134,9 @@ void deleteFromStore(const Path & _path, unsigned long long & bytesFreed) if (*i != path && isValidPathTxn(txn, *i)) throw PathInUse(format("cannot delete path `%1%' because it is in use by path `%2%'") % path % *i); invalidatePath(txn, path); + + //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + //Also delete stateReferrers????? } txn.commit(); @@ -1178,6 +1227,7 @@ void verifyStore(bool checkContents) } } + //TODO also check state references and refererres /* Check the `references' table. */ printMsg(lvlInfo, "checking the references table"); @@ -1245,6 +1295,7 @@ void verifyStore(bool checkContents) queryReferences(txn, from, references); PathSet stateReferences; + queryStateReferences(txn, from, stateReferences); //TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if (find(references.begin(), references.end(), to) == references.end()) { @@ -1257,7 +1308,7 @@ void verifyStore(bool checkContents) } - //TODO Check statecounters table.... + //TODO Check stateinfo and statecounters table txn.commit(); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 09194c32c..080d466d3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -50,6 +50,8 @@ public: void queryStateReferences(const Path & storePath, PathSet & stateReferences); void queryReferrers(const Path & path, PathSet & referrers); + + void queryStateReferrers(const Path & path, PathSet & stateReferrers); Path addToStore(const Path & srcPath, bool fixed = false, bool recursive = false, string hashAlgo = "", @@ -84,7 +86,9 @@ public: bool isStateComponent(const Path & path); - bool isStateDrv(const Path & drvpath); + bool isStateDrvPath(const Path & drvpath); + + bool isStateDrv(Derivation drv); }; @@ -140,7 +144,7 @@ void canonicalisePathMetaData(const Path & path); /* Checks whether a path is valid. */ bool isValidPathTxn(const Transaction & txn, const Path & path); -/* Sets the set of outgoing FS references for a store path. Use with +/* Sets the set of outgoing FS (also state) references for a store path. Use with care! */ void setReferences(const Transaction & txn, const Path & path, const PathSet & references, const PathSet & stateReferences); @@ -190,6 +194,10 @@ void addStateDeriver(const Transaction & txn, const Path & storePath, const Path PathSet mergeNewDerivationIntoList(const Path & storepath, const Path & newdrv, const PathSet drvs, bool deleteDrvs = false); bool isStateComponentTxn(const Transaction & txn, const Path & path); + +bool isStateDrvPathTxn(const Transaction & txn, const Path & drvPath); + +bool isStateDrvTxn(const Transaction & txn, Derivation drv); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3cdab7661..09c52454d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -206,10 +206,8 @@ void RemoteStore::queryStateReferences(const Path & path, writeInt(wopQueryStateReferences, to); writeString(path, to); processStderr(); - - //TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -// PathSet references2 = readStorePaths(from); -// references.insert(references2.begin(), references2.end()); + PathSet stateReferences2 = readStorePaths(from); + stateReferences.insert(stateReferences2.begin(), stateReferences2.end()); } @@ -224,6 +222,15 @@ void RemoteStore::queryReferrers(const Path & path, referrers.insert(referrers2.begin(), referrers2.end()); } +void RemoteStore::queryStateReferrers(const Path & path, PathSet & stateReferrers) +{ + writeInt(wopQueryStateReferrers, to); + writeString(path, to); + processStderr(); + PathSet stateReferrers2 = readStorePaths(from); + stateReferrers.insert(stateReferrers2.begin(), stateReferrers2.end()); +} + Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, bool recursive, string hashAlgo, PathFilter & filter) @@ -402,15 +409,22 @@ vector RemoteStore::getStatePathsInterval(const PathSet & statePaths) return intervals; } +//TODO bool RemoteStore::isStateComponent(const Path & path) { return false; } -bool RemoteStore::isStateDrv(const Path & drvpath) +//TODO +bool RemoteStore::isStateDrvPath(const Path & drvpath) { - return false; + return false; } +//TODO +bool RemoteStore::isStateDrv(Derivation drv) +{ + return false; +} } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 520df21af..95589bead 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -38,6 +38,8 @@ public: void queryStateReferences(const Path & storePath, PathSet & stateReferences); void queryReferrers(const Path & path, PathSet & referrers); + + void queryStateReferrers(const Path & path, PathSet & stateReferrers); Path addToStore(const Path & srcPath, bool fixed = false, bool recursive = false, string hashAlgo = "", @@ -72,7 +74,9 @@ public: bool isStateComponent(const Path & path); - bool isStateDrv(const Path & drvpath); + bool isStateDrvPath(const Path & drvpath); + + bool isStateDrv(Derivation drv); private: diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 77959a008..f7b34fc68 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -83,6 +83,10 @@ public: virtual void queryReferrers(const Path & path, PathSet & referrers) = 0; + /* Queries the set of incoming FS state-references for a store path. + The result is not cleared. */ + virtual void queryStateReferrers(const Path & path, PathSet & stateReferrers) = 0; + /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. If `fixed' is true, then the output of a fixed-output @@ -196,7 +200,10 @@ public: virtual bool isStateComponent(const Path & path) = 0; /* TODO */ - virtual bool isStateDrv(const Path & drvpath) = 0; + virtual bool isStateDrvPath(const Path & drvpath) = 0; + + /* TODO */ + virtual bool isStateDrv(Derivation drv) = 0; }; diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index 7b09e2b5a..f84d557ca 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -147,6 +147,17 @@ static void opShowStateReposRootPath(Strings opFlags, Strings opArgs) } +//TODO +static void recheckrefsinstaterecursive() +{ + + //PathSet state_references = scanForReferences(statePath, allPaths); + //PathSet state_stateReferences = scanForStateReferences(statePath, allStatePaths); + +} + + + static void opRunComponent(Strings opFlags, Strings opArgs) { //get the derivation of the current component @@ -356,7 +367,7 @@ void run(Strings args) store = openStore(); printMsg(lvlError, format("1: %1%") % bool2string( store->isStateComponent("/nix/store/7xkw5fkz5yw7dpx0pc6l12bh9a56135c-hellostateworld-1.0") ) ); printMsg(lvlError, format("2: %1%") % bool2string( store->isStateComponent("/nix/store/05441jm8xmsidqm43ivk0micckf0mr2m-nvidiaDrivers") ) ); - printMsg(lvlError, format("3: %1%") % bool2string( store->isStateDrv("/nix/store/2hpx60ibdfv2pslg4rjvp177frijamvi-hellostateworld-1.0.drv") ) ); + printMsg(lvlError, format("3: %1%") % bool2string( store->isStateDrvPath("/nix/store/2hpx60ibdfv2pslg4rjvp177frijamvi-hellostateworld-1.0.drv") ) ); return; /* test */ @@ -394,6 +405,11 @@ void run(Strings args) --user=... + --show-state-references- -rev = ... + --show-state-references-current //in nix-store + --show-state-referrers- -rev = ... + --show-state-referrers-current //in nix-store + */ else diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index 7355835d0..058dfb699 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -34,6 +34,7 @@ Query flags: --references: print all paths referenced by the given path --references-state: print all state paths referenced by the given path --referrers: print all paths directly refering to the given path + --referrers-state: print all state paths directly refering to the given path --referrers-closure: print all paths (in)directly refering to the given path --tree: print a tree showing the dependency graph of the given paths --graph: print a dot graph rooted at given paths diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index d41daf0df..339f6b30a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -257,7 +257,7 @@ static void printTree(const Path & path, /* Perform various sorts of queries. */ static void opQuery(Strings opFlags, Strings opArgs) { - enum { qOutputs, qRequisites, qReferences, qStateReferences, qReferrers + enum { qOutputs, qRequisites, qReferences, qStateReferences, qReferrers, qStateReferrers , qReferrersClosure, qDeriver, qBinding, qHash , qTree, qGraph, qResolve } query = qOutputs; bool useOutput = false; @@ -272,6 +272,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (*i == "--references") query = qReferences; else if (*i == "--references-state") query = qStateReferences; else if (*i == "--referrers" || *i == "--referers") query = qReferrers; + else if (*i == "--referrers-state" || *i == "--referers-state") query = qStateReferrers; else if (*i == "--referrers-closure" || *i == "--referers-closure") query = qReferrersClosure; else if (*i == "--deriver" || *i == "-d") query = qDeriver; else if (*i == "--binding" || *i == "-b") { @@ -308,6 +309,7 @@ static void opQuery(Strings opFlags, Strings opArgs) case qReferences: case qStateReferences: case qReferrers: + case qStateReferrers: case qReferrersClosure: { PathSet paths; for (Strings::iterator i = opArgs.begin(); @@ -319,6 +321,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (query == qReferences) store->queryReferences(path, paths); else if (query == qStateReferences) store->queryStateReferences(path, paths); else if (query == qReferrers) store->queryReferrers(path, paths); + else if (query == qStateReferrers) store->queryStateReferrers(path, paths); else if (query == qReferrersClosure) computeFSClosure(path, paths, true); } Paths sorted = topoSortPaths(paths); diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index b390eb4b9..f7bda6e0d 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -284,8 +284,8 @@ static void performOp(Source & from, Sink & to, unsigned int op) PathSet paths; if (op == wopQueryStateReferences) store->queryStateReferences(path, paths); - //else - // store->queryStateReferrers(path, paths); //TODO TODO implemnt function, and then commen out !!!!!!!!!!!!!!!!!!!!! + else + store->queryStateReferrers(path, paths); //TODO Does this work??? stopWork(); writeStringSet(paths, to); break;