From 138973a6d543ed596b38110da5b458675012971f Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Tue, 16 Oct 2007 17:11:18 +0000 Subject: [PATCH] setSnapshot & build-stateinfo fixes --- src/libstore/build.cc | 8 +++- src/libstore/gc.cc | 33 ++++++--------- src/libstore/local-store.cc | 51 +++++++---------------- src/libstore/local-store.hh | 1 + src/libstore/store-state.cc | 79 +++++++++++++++++++++++++++++------- src/libstore/store-state.hh | 12 ++++-- src/libutil/types.hh | 2 +- src/nix-state/nix-state.cc | 11 +++-- src/nix-worker/nix-worker.cc | 6 +-- 9 files changed, 121 insertions(+), 82 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 275e19676..f506a8c29 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1889,7 +1889,11 @@ void DerivationGoal::computeClosure() for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) { - registerValidPath(txn, + //Register the path as a store-state path + if(isStateDrvPathTxn(txn, drvPath)) + setStateComponentTxn(txn, i->second.path); + + registerValidPath(txn, i->second.path, //component path contentHashes[i->second.path], allReferences[i->second.path], //set of component-references @@ -1901,7 +1905,7 @@ void DerivationGoal::computeClosure() if(isStateDrv(drv)) { Path statePath = drv.stateOutputs.find("state")->second.statepath; - + registerValidPath(txn, statePath, Hash(), //emtpy hash diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index fd459d203..d82118ba7 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -479,46 +479,41 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, /* Determine the live paths which is just the closure of the roots under the `references' relation. */ PathSet livePaths; - for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i){ - //printMsg(lvlError, format("CHECK '%1%'") % *i); + for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i) computeFSClosure(canonPath(*i), livePaths, true, true, 0); - } printMsg(lvlError, format("STAGE X")); + /* Note that the deriver need not be valid (e.g., if we + previously ran the collector with `gcKeepDerivations' + turned off). */ if (gcKeepDerivations) { for (PathSet::iterator i = livePaths.begin(); i != livePaths.end(); ++i) { if (store->isStateComponent(*i)){ //printMsg(lvlError, format("Live state store '%1%'") % *i); - //we select ALL state Derivations here - PathSet derivers = store->queryDerivers(*i, "*", "*"); - + PathSet derivers = store->queryDerivers(*i, "*", "*"); //we select ALL state Derivations here //TODO ??? we shouldt select non live !!!!!!!!! for (PathSet::const_iterator j = derivers.begin(); j != derivers.end(); ++j) - // We send each drv to computeFSClosure if (*j != "" && store->isValidPath(*j)) computeFSClosure(*j, livePaths, true, true, 0); } else if (store->isValidPath(*i)){ //printMsg(lvlError, format("Live Store '%1%'") % *i); - Path deriver = store->queryDeriver(*i); - /* Note that the deriver need not be valid (e.g., if we - previously ran the collector with `gcKeepDerivations' - turned off). */ + Path deriver = store->queryDeriver(*i); if (deriver != "" && store->isValidPath(deriver)) computeFSClosure(deriver, livePaths, true, true, 0); } else if (store->isValidStatePath(*i)){ - //printMsg(lvlError, format("Live State '%1%'") % *i); + //printMsg(lvlError, format("Live State '%1%'") % *i); Path deriver = queryStatePathDrvTxn(noTxn, *i); - //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! TODO put back on + //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! TODO put back on if(!store->isValidPath(deriver)) {} // throw Error(format("deriver `%1%' of state-store component `%2%' in GC is not valid") % deriver % *i); - else //TODO !!!!!!!!!!!!!! REMOVE ELSE - computeFSClosure(deriver, livePaths, true, true, 0); + + computeFSClosure(deriver, livePaths, true, true, 0); } } } @@ -529,7 +524,6 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, /* Hmz, identical to storePathRequisites in nix-store. */ for (PathSet::iterator i = livePaths.begin(); i != livePaths.end(); ++i) if (isDerivation(*i)) { - //printMsg(lvlError, format("CHECK3 '%1%'") % *i); Derivation drv = derivationFromPathTxn(noTxn, *i); for (DerivationOutputs::iterator j = drv.outputs.begin(); j != drv.outputs.end(); ++j) @@ -577,10 +571,9 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, can be deleted. */ /* - * We lookup all shared paths for: livePaths, tempRootsClosed + * We lookup all state (also shared) paths for: livePaths, tempRootsClosed */ PathSet allLiveStatePaths; - for (PathSet::iterator i = livePaths.begin(); i != livePaths.end(); ++i) if(store->isValidStatePath(*i)){ allLiveStatePaths.insert(*i); @@ -593,7 +586,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, } /* - * Lookup all derivations, of all state paths, because they need to be kept for comitting + * Lookup all derivations, of all state paths, because they need to be kept for comitting //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ PathSet allStatePathDerivations; for (PathSet::iterator i = allLiveStatePaths.begin(); i != allLiveStatePaths.end(); ++i){ @@ -629,7 +622,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, for (PathSet::iterator i = pathsToDelete.begin(); i != pathsToDelete.end(); ++i) { - assertStorePath(*i); //TODO ASSERTSTATEPATH, we for now we have no arg statePathsToDelete + assertStorePath(*i); //TODO ASSERTSTATEPATH, but for now we have no arg statePathsToDelete storePathSet.insert(*i); } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3d505dff4..e9cb6eaab 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -129,11 +129,11 @@ static TableId dbStateRevisionsComments = 0; * * This table stores the snapshot numbers the sub state files/folders * at a certain timestamp. These snapshot numbers are just timestamps - * produced by ext3cow + * produced by ext3cow * * /nix/state/HASH-A-1.0-test-KEY-1185473750 * --> - * [ 1185473750, 00118547375 ] + * [ 1185473750, 00118547375 ] // TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! PATH-SS * * The timestamps are ordered based on the path of the subfolder !! * @@ -708,10 +708,12 @@ void addStateDeriver(const Transaction & txn, const Path & storePath, const Path data.push_back(*i); nixDB.setStrings(txn, dbDerivers, storePath, data); //update the derivers db. - - nixDB.setString(txn, dbStateInfo, storePath, ""); //update the dbinfo db. (maybe TODO) } +void setStateComponentTxn(const Transaction & txn, const Path & storePath) +{ + nixDB.setString(txn, dbStateInfo, storePath, ""); //update the dbinfo db. (maybe TODO) +} /* * Returns true or false wheter a store-component has a state component (e.g. has a state dir) or not. @@ -979,41 +981,18 @@ static void invalidatexPath(Transaction & txn, const Path & path, const bool com //if is store-state if(isStateComponentTxn(txn, path)){ - - //delete all references - + nixDB.delPair(txn, dbSolidStateReferences, path); + nixDB.delPair(txn, dbStateInfo, path); } - - nixDB.delPair(txn, dbStateInfo, path); //We (may) need to delete if this key if path is a state-store path - - - //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! } else{ - - //For now.... we only delete clear references for the new revision (TODO !!!!!!!!!!!!!!!! CHECK IF WE REALLY SET IT FOR A NEW OR LAST REVISION) - setReferences(txn, path, PathSet(), PathSet(), 0); - nixDB.delPair(txn, dbDerivers, path); + + nixDB.delPair(txn, dbDerivers, path); nixDB.delPair(txn, dbValidStatePaths, path); - - //TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! also clear - - //state-counters - ///nix/state/pspj0srf5vh6qhwa1i11swqai9lnk581-hellohardcodedstateworld-1.0-test/log/ - //0 - - //revisions - ///nix/state/4ngfrqhfj7n28p334ajcji9zrdrbdgaw-hellohardcodedstateworld-1.0-test - //1 - ///nix/state/aacs4qpi9jzg4vmhj09d0ichframh22x-hellohardcodedstateworld-1.0-test-KEY-16 - - //comments - // /nix/state/ryxrs01hkxyk4x96i58pc7n54y1gyhmb-hellostateworld-1.0-test-KEY-1 - // \0a\00\00\001187346592\17\00\00\00Initial build revision. - - //stateSnapshots - //nix/state/54crxisppkqy7929dysqyjdiladahrw7-hellohardcodedstateworld-1.0-test-KEY-1189076361 - //\01\00\00\000\0a\00\00\001189076361\01\00\00\000 + + invalidateAllStateReferences(nixDB, txn, dbStateComponentReferences, path); + invalidateAllStateReferences(nixDB, txn, dbStateStateReferences, path); + removeAllStatePathRevisions(nixDB, txn, dbStateRevisions, dbStateRevisionsComments, dbStateSnapshots, dbStateCounters, path); } } @@ -1330,7 +1309,7 @@ void deleteXFromStore(const Path & _path, unsigned long long & bytesFreed, const Transaction txn(nixDB); if (isValidPathTxn(txn, path) || isValidStatePathTxn(txn, path)) { - PathSet storeReferrers = getStoreReferrersTxn(txn, path, 0); + PathSet storeReferrers = getStoreReferrersTxn(txn, path, 0); //TODO GET REFERRERS OF ALL REVISIONS !!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!! PathSet stateReferrers = getStateReferrersTxn(txn, path, 0); for (PathSet::iterator i = storeReferrers.begin(); i != storeReferrers.end(); ++i) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 13f8c644d..edc20f28d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -273,6 +273,7 @@ bool queryStateRevisionsTxn(const Transaction & txn, const Path & statePath, Rev void setStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePath, const IntVector & intervals, bool allZero = false); bool querySharedStateTxn(const Transaction & txn, const Path & statePath, Path & shared_with); +void setStateComponentTxn(const Transaction & txn, const Path & storePath); } diff --git a/src/libstore/store-state.cc b/src/libstore/store-state.cc index 27c5dbba4..69f60b261 100644 --- a/src/libstore/store-state.cc +++ b/src/libstore/store-state.cc @@ -128,18 +128,21 @@ void revertToRevisionTxn(const Transaction & txn, const Path & statePath, const Snapshots revisioned_paths = (*i).second; unsigned int timestamp = getTimestamps[statePath]; + //get its derivation-state-items - Derivation statePath_drv = derivationFromPathTxn(txn, queryStatePathDrvTxn(txn, statePath)); - DerivationStateOutputDirs stateOutputDirs = statePath_drv.stateOutputDirs; + //Derivation statePath_drv = derivationFromPathTxn(txn, queryStatePathDrvTxn(txn, statePath)); + //DerivationStateOutputDirs stateOutputDirs = statePath_drv.stateOutputDirs; //TODO Sort snapshots??? eg first restore root, then the subdirs?? for (Snapshots::iterator j = revisioned_paths.begin(); j != revisioned_paths.end(); ++j){ Path revertPathOrFile = (*j).first; - unsigned int epoch = (*j).second; + unsigned int epoch = (*j).second; //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - //printMsg(lvlError, format("MAYBE '%1%'") % revertPathOrFile); + printMsg(lvlError, format("MAYBE '%1%''%2%'") % revertPathOrFile % epoch); + + /* //Look up the type from the drv with for the current snapshotted path Path statePath_postfix = revertPathOrFile.substr(nixStoreState.length() + 1, revertPathOrFile.length() - nixStoreState.length()); statePath_postfix = statePath_postfix.substr(statePath_postfix.find_first_of("/") + 1, statePath_postfix.length()); @@ -152,8 +155,9 @@ void revertToRevisionTxn(const Transaction & txn, const Path & statePath, const //Now that were still here, we need to copy the state from the previous version back Strings p_args; p_args.push_back("-c"); //we use the shell to execute the cp command becuase the shell expands the '*' - string cpcommand = "cp"; - + string cpcommand = "cp"; //TODO USE RSYNC!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + */ + /* if(revertPathOrFile.substr(revertPathOrFile.length() -1 , revertPathOrFile.length()) == "/"){ //is dir string revert_to_path = revertPathOrFile.substr(0, revertPathOrFile.length() -1) + "@" + unsignedInt2String(epoch); cpcommand += " -R " + revert_to_path + "/*"; @@ -192,11 +196,12 @@ void revertToRevisionTxn(const Transaction & txn, const Path & statePath, const cpcommand += " " + revertPathOrFile; p_args.push_back(cpcommand); runProgram_AndPrintOutput("sh", true, p_args, "sh-cp"); + */ } //*** Now also revert state _references_ to the specific revision (the revision is already converted to a timestamp here) - + /* //Query the references of the old revision (already converted to a timestamp) PathSet state_references; queryXReferencesTxn(txn, statePath, state_references, true, 0, timestamp); @@ -211,6 +216,7 @@ void revertToRevisionTxn(const Transaction & txn, const Path & statePath, const printMsg(lvlError, format("Reverted state of '%1%' to the latest revision") % statePath); //TODO lookup the number else printMsg(lvlError, format("Reverted state of '%1%' to revision '%2%'") % statePath % revision_arg); + */ } } @@ -284,6 +290,7 @@ Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath) revision_number = 0; //deleted, so we assign 0 to indicate that revisions_list[fullstatedir] = revision_number; + //printMsg(lvlError, format("FSD %1% RN %2%") % fullstatedir % revision_number); } return revisions_list; @@ -574,6 +581,49 @@ bool queryStateReferences(Database & nixDB, const Transaction & txn, TableId ref return nixDB.queryStrings(txn, references_table, key, references); //now that we have the key, we can query the references } +void invalidateAllStateReferences(Database & nixDB, const Transaction & txn, TableId references_table, const Path & statePath) +{ + //Remove all references + Strings keys; + nixDB.enumTable(txn, references_table, keys); + for (Strings::const_iterator i = keys.begin(); i != keys.end(); ++i){ + if((*i).substr(0, statePath.length()) == statePath) + nixDB.delPair(txn, references_table, *i); + } +} + +void removeAllStatePathRevisions(Database & nixDB, const Transaction & txn, TableId revisions_table, + TableId revisions_comments, TableId snapshots_table, TableId statecounters, const Path & statePath) +{ + //Remove all revisions + nixDB.delPair(txn, revisions_table, statePath); + RevisionInfos revisions; + store->queryAvailableStateRevisions(statePath, revisions); + for (RevisionInfos::iterator i = revisions.begin(); i != revisions.end(); ++i){ + unsigned int rev = (*i).first; + nixDB.delPair(txn, revisions_table, mergeToDBKey(statePath, rev)); + + //Remove all comments + nixDB.delPair(txn, revisions_comments, mergeToDBKey(statePath, rev)); + } + + //Remove all snapshots + Strings keys; + nixDB.enumTable(txn, snapshots_table, keys); + for (Strings::const_iterator i = keys.begin(); i != keys.end(); ++i){ + if((*i).substr(0, statePath.length()) == statePath) + nixDB.delPair(txn, snapshots_table, *i); + } + + //Remove all state-counters + keys.clear(); + nixDB.enumTable(txn, statecounters, keys); + for (Strings::const_iterator i = keys.begin(); i != keys.end(); ++i){ + if((*i).substr(0, statePath.length()) == statePath) + nixDB.delPair(txn, statecounters, *i); + } +} + void setStateRevisions(Database & nixDB, const Transaction & txn, TableId revisions_table, TableId revisions_comments, TableId snapshots_table, const RevisionClosure & revisions, const Path & rootStatePath, const string & comment) { @@ -588,7 +638,7 @@ void setStateRevisions(Database & nixDB, const Transaction & txn, TableId revisi Strings data; //the map<> takes care of the sorting on the Path for (Snapshots::const_iterator j = (*i).second.begin(); j != (*i).second.end(); ++j) - data.push_back(int2String((*j).second)); + data.push_back(mergeToDBKey((*j).first, (*j).second)); nixDB.setStrings(txn, snapshots_table, key, data); } @@ -657,6 +707,7 @@ bool queryStateRevisions(Database & nixDB, const Transaction & txn, TableId revi splitDBKey(*i, getStatePath, getTimestamp); //query state versioined directorys/files + /* vector sortedPaths; Derivation drv = derivationFromPathTxn(txn, queryStatePathDrvTxn(txn, getStatePath)); DerivationStateOutputs stateOutputs = drv.stateOutputs; @@ -669,6 +720,7 @@ bool queryStateRevisions(Database & nixDB, const Transaction & txn, TableId revi sortedPaths.push_back(fullstatedir); } sort(sortedPaths.begin(), sortedPaths.end()); //sort + */ Strings snapshots_s; Snapshots snapshots; @@ -676,12 +728,11 @@ bool queryStateRevisions(Database & nixDB, const Transaction & txn, TableId revi int counter=0; for (Strings::iterator j = snapshots_s.begin(); j != snapshots_s.end(); ++j){ + Path snapshottedPath; unsigned int revision; - bool succeed = string2UnsignedInt(*j, revision); - if(!succeed) - throw Error(format("Malformed epoch (snapshot timestamp) value of path '%1%'") % statePath); - - snapshots[sortedPaths.at(counter)] = revision; + splitDBKey(*j, snapshottedPath, revision); + + snapshots[snapshottedPath] = revision; counter++; } @@ -715,7 +766,7 @@ bool queryAvailableStateRevisions(Database & nixDB, const Transaction & txn, Tab unsigned int ts; bool succeed = string2UnsignedInt(*(metadata.begin()), ts); if(!succeed) - throw Error(format("Malformed timestamp in the revisions-comments table of path '%1%'") % *i); + throw Error(format("Malformed timestamp in the revisions table of path '%1%'") % *i); rev.timestamp = ts; metadata.pop_front(); rev.comment = *(metadata.begin()); diff --git a/src/libstore/store-state.hh b/src/libstore/store-state.hh index 317d586e9..c9f54f9a4 100644 --- a/src/libstore/store-state.hh +++ b/src/libstore/store-state.hh @@ -28,6 +28,9 @@ namespace nix { /* revision 0 == latest ????? */ void revertToRevisionTxn(const Transaction & txn, const Path & statePath, const int revision_arg, const bool recursive); + /* Copy all files and folders recursively (also the hidden ones) from the dir from/... to the dir to/... and delete the rest in to/ (Rsync) */ + void rsyncPaths(const Path & from, const Path & to); + // **************************************** ******************************************* @@ -67,9 +70,12 @@ namespace nix { bool queryAvailableStateRevisions(Database & nixDB, const Transaction & txn, TableId revisions_table, TableId revisions_comments, const Path & statePath, RevisionInfos & revisions); - /* Copy all files and folders recursively (also the hidden ones) from the dir from/... to the dir to/... and delete the rest in to/ (Rsync) */ - void rsyncPaths(const Path & from, const Path & to); - + void invalidateAllStateReferences(Database & nixDB, const Transaction & txn, TableId references_table, const Path & statePath); + + void removeAllStatePathRevisions(Database & nixDB, const Transaction & txn, TableId revisions_table, + TableId revisions_comments, TableId snapshots_table, TableId statecounters, const Path & statePath); + + } #endif /* !__STORESTATE_H */ diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 5af12f6b3..af5f9828f 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -71,7 +71,7 @@ struct RevisionInfo unsigned int timestamp; }; typedef map RevisionInfos; -typedef map Snapshots; //Automatically sorted on Path :) +typedef map Snapshots; // /nix/state/...../cache -> .... Automatically sorted on Path :) typedef map RevisionClosure; typedef map RevisionClosureTS; //Paht with a timestamp about when the revision was made. typedef map StateReferences; diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index 63087f279..318917587 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -70,7 +70,7 @@ Derivation getDerivation(const string & fullPath, const Strings & program_args, //Check if path is store-statepath isStateComponent = store->isStateComponent(componentPath); - //printMsg(lvlError, format("'%1%' - '%2%' - '%3%' - '%4%' - '%5%'") % componentPath % state_identifier % binary % username % program_args); + printMsg(lvlError, format("'%1%' - '%2%' - '%3%' - '%4%' - '%5%'") % componentPath % state_identifier % binary % username % bool2string(isStateComponent)); if(isStateComponent) derivers = store->queryDerivers(componentPath, state_identifier, username); @@ -240,9 +240,9 @@ static void queryAvailableStateRevisions(Strings opFlags, Strings opArgs) string comment = revisions[rev].comment; if(trim(comment) != "") - printMsg(lvlError, format("Rev. %1% @ %2% (%3%) -- %4%") % rev_s % human_date % ts % comment); + printMsg(lvlError, format("Rev. %1% @ %2% (ts: %3%) -- %4%") % rev_s % human_date % ts % comment); else - printMsg(lvlError, format("Rev. %1% @ %2% (%3%)") % rev_s % human_date % ts); + printMsg(lvlError, format("Rev. %1% @ %2% (ts: %3%)") % rev_s % human_date % ts); } } @@ -596,6 +596,11 @@ void run(Strings args) */ + /* + store = openStore(); + store->isStateComponent("/nix/state/rrki0fgjc42sfszgk95cg0bpchbc5xp7-hellohardcodedstateworld-1.0-test"); + return; + */ /* test */ if(args.size() == 1 && ( *(args.begin()) == "--help" || *(args.begin()) == "--statehelp")){ diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index e2b3641c2..270b4fb5e 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -705,10 +705,10 @@ static void processConnection() int oppp = readInt(from); op = (WorkerOp) oppp; - /* Use for debugging*/ + /* Use for debugging with gdb --pid=myPid */ /* - if(oppp == 14){ - printMsg(lvlError, format("Sleeping 10")); + if(oppp == 39){ + printMsg(lvlError, format("Sleeping 10 before op '%1%' with pid '%2%'") % op % myPid); sleep(10); } */