From 0fc5accd86ca038b62172661942f8683f272e427 Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Wed, 25 Jul 2007 21:52:33 +0000 Subject: [PATCH] Replaced SVN by Ext3COW as a backend for state (still some things need to happen: reverting doesn't go right in all cases yet) --- src/libext3cow/snapshot.cc | 2 +- src/libstore/build.cc | 8 +- src/libstore/db.cc | 197 +++++++++++++++++------------------ src/libstore/db.hh | 10 +- src/libstore/local-store.cc | 28 ++--- src/libstore/local-store.hh | 6 +- src/libstore/remote-store.cc | 4 +- src/libstore/remote-store.hh | 4 +- src/libstore/store-api.cc | 23 ---- src/libstore/store-api.hh | 7 +- src/libstore/store-state.cc | 90 +++------------- src/libstore/store-state.hh | 2 +- src/libutil/types.hh | 5 +- src/libutil/util.cc | 29 +++++- src/libutil/util.hh | 12 ++- src/nix-state/nix-state.cc | 110 +++++++++++++++---- 16 files changed, 269 insertions(+), 268 deletions(-) diff --git a/src/libext3cow/snapshot.cc b/src/libext3cow/snapshot.cc index 18789d4ba..4b9ab6c87 100644 --- a/src/libext3cow/snapshot.cc +++ b/src/libext3cow/snapshot.cc @@ -89,7 +89,7 @@ unsigned int take_snapshot(const string & dir2) //const string & file_or_dir) exit(1); } - printf("%u\n", (unsigned int)epoch); + //printf("%u\n", (unsigned int)epoch); return epoch; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2bdb476ab..33409aeef 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1809,8 +1809,12 @@ void DerivationGoal::computeClosure() state_stateReferences, drvPath, 0); - //Commit state - commitStatePathTxn(txn, statePath); + //Commit state (we only include our own state in the rivisionMapping (but other build component states might have been changed !!!! TODO) + RevisionClosure rivisionMapping; + rivisionMapping[statePath] = commitStatePathTxn(txn, statePath); + + //Save the new revision + setStateRevisionsTxn(txn, statePath, rivisionMapping); //Shared state Path sharedState = drv.stateOutputs.find("state")->second.sharedState; diff --git a/src/libstore/db.cc b/src/libstore/db.cc index b1afc4c8a..c4e4b07c9 100644 --- a/src/libstore/db.cc +++ b/src/libstore/db.cc @@ -466,7 +466,6 @@ void Database::splitStatePathRevision(const Path & revisionedStatePath, Path & s if(!succeed) throw Error(format("Malformed revision value of path '%1%'") % revisionedStatePath); } - int Database::getNewRevisionNumber(const Transaction & txn, TableId table, const Path & statePath) @@ -491,31 +490,6 @@ int Database::getNewRevisionNumber(const Transaction & txn, TableId table, return revision; } -void Database::setStateReferences(const Transaction & txn, TableId table, - const Path & statePath, const Strings & references, int revision) -{ - //printMsg(lvlError, format("setStateReferences/Referrers %1%") % table); - - if(revision == -1) - revision = getNewRevisionNumber(txn, table, statePath); - - /* - for (Strings::const_iterator i = references.begin(); i != references.end(); ++i) - printMsg(lvlError, format("setStateReferences::: '%1%'") % *i); - */ - - //Warning if it already exists - Strings empty; - if( queryStateReferences(txn, table, statePath, empty, revision) ) - printMsg(lvlError, format("Warning: The revision '%1%' already exists for set-references/referrers of path '%2%' with db '%3%'") % revision % statePath % table); - - //Create the key - string key = makeStatePathRevision(statePath, revision); - - //Insert - setStrings(txn, table, key, references); -} - bool Database::lookupHighestRevivison(const Strings & keys, const Path & statePath, string & key, int lowerthan) { int highestRev = -1; @@ -548,6 +522,33 @@ bool Database::lookupHighestRevivison(const Strings & keys, const Path & statePa return true; } +////////////////////////////////////////////// + +void Database::setStateReferences(const Transaction & txn, TableId table, + const Path & statePath, const Strings & references, int revision) +{ + //printMsg(lvlError, format("setStateReferences/Referrers %1%") % table); + + if(revision == -1) + revision = getNewRevisionNumber(txn, table, statePath); + + /* + for (Strings::const_iterator i = references.begin(); i != references.end(); ++i) + printMsg(lvlError, format("setStateReferences::: '%1%'") % *i); + */ + + //Warning if it already exists + Strings empty; + if( queryStateReferences(txn, table, statePath, empty, revision) ) + printMsg(lvlError, format("Warning: The revision '%1%' already exists for set-references/referrers of path '%2%' with db '%3%'") % revision % statePath % table); + + //Create the key + string key = makeStatePathRevision(statePath, revision); + + //Insert + setStrings(txn, table, key, references); +} + bool Database::queryStateReferences(const Transaction & txn, TableId table, const Path & statePath, Strings & references, int revision) { @@ -589,129 +590,119 @@ bool Database::queryStateReferrers(const Transaction & txn, TableId table, //Exactly the same as queryStateReferences return queryStateReferences(txn, table, statePath, referrers, revision); } + +///////////////////////////////////////////// -void Database::setStateRevisions(const Transaction & txn, TableId table, - const Path & statePath, const RevisionNumbersSet & revisions) +void Database::setStateRevisions(const Transaction & txn, TableId revisions_table, TableId snapshots_table, + const Path & root_statePath, const RevisionClosure & revisions) { - //get a new revision number - int root_revision = getNewRevisionNumber(txn, table, statePath); + int ts = getTimeStamp(); - //Sort based on statePath to RevisionNumbersClosure - vector sortedStatePaths; - for (RevisionNumbersSet::const_iterator i = revisions.begin(); i != revisions.end(); ++i) - sortedStatePaths.push_back((*i).first); - sort(sortedStatePaths.begin(), sortedStatePaths.end()); - - vector sorted_revisions; - for (vector::const_iterator i = sortedStatePaths.begin(); i != sortedStatePaths.end(); ++i){ - map ss_revisions = revisions.at(*i); - - //Sort the set of paths that have revisions based on - vector sorted_ssp; - for (map::const_iterator j = ss_revisions.begin(); j != ss_revisions.end(); ++j) - sorted_ssp.push_back((*j).first); - sort(sorted_ssp.begin(), sorted_ssp.end()); - - //Insert into sorted_ss_revs based on the sorted order - RevisionNumbers sorted_ss_revs; - for (vector::const_iterator j = sorted_ssp.begin(); j != sorted_ssp.end(); ++j) - sorted_ss_revs.push_back(ss_revisions.at(*j)); - - //Insert ...... - sorted_revisions.push_back(sorted_ss_revs); - } + //Insert all ss_epochs into table ...... with the current ts. + for (RevisionClosure::const_iterator i = revisions.begin(); i != revisions.end(); ++i){ + string key = makeStatePathRevision((*i).first, ts); + Strings data; + //the map<> takes care of the sorting on + for (Snapshots::const_iterator j = (*i).second.begin(); j != (*i).second.end(); ++j) + data.push_back(int2String((*j).second)); + setStrings(txn, snapshots_table, key, data); + } - //Debugging - //for (vector::const_iterator i = sortedStatePaths.begin(); i != sortedStatePaths.end(); ++i) - // printMsg(lvlError, format("Insert: %1% into %2%") % revisions.at(*i) % *i); + //Insert for each statePath a new revision record linked to the ss_epochs + for (RevisionClosure::const_iterator i = revisions.begin(); i != revisions.end(); ++i){ + Path statePath = (*i).first; + int revision = getNewRevisionNumber(txn, revisions_table, statePath); //get a new revision number + string key = makeStatePathRevision(statePath, revision); - //Convert the vector into Strings - Strings data; - for (vector::const_iterator i = sorted_revisions.begin(); i != sorted_revisions.end(); ++i) - data.push_back(packRevisionNumbers(*i)); + //get all its requisites + PathSet statePaths; + storePathRequisitesTxn(txn, statePath, false, statePaths, false, true, -1); + statePaths.insert(statePath); - //Create the key - string key = makeStatePathRevision(statePath, root_revision); - - //Insert - setStrings(txn, table, key, data); + //save in db + Strings data; + for (PathSet::const_iterator j = statePaths.begin(); j != statePaths.end(); ++j) + data.push_back(makeStatePathRevision(*j, ts)); + setStrings(txn, revisions_table, key, data); + } } -bool Database::queryStateRevisions(const Transaction & txn, TableId table, const PathSet statePath_deps, - const Path & statePath, RevisionNumbersSet & revisions, int root_revision) +bool Database::queryStateRevisions(const Transaction & txn, TableId revisions_table, TableId snapshots_table, + const Path & statePath, RevisionClosure & revisions, int root_revision) { - Strings keys; - enumTable(txn, table, keys); //get all revisions - string key; + if(root_revision == -1){ + Strings keys; + enumTable(txn, revisions_table, keys); //get all revisions bool foundsomething = lookupHighestRevivison(keys, statePath, key); if(!foundsomething) return false; } else key = makeStatePathRevision(statePath, root_revision); - - Strings data; - bool notempty = queryStrings(txn, table, key, data); //now that we have the key, we can query the revisions - - //Check - if(statePath_deps.size() != data.size()) - throw Error(format("The number of statepath references doenst equal the number of revisions for '%1%'") % statePath); - - //sort all state references recursively - vector sortedStatePaths; - for (PathSet::iterator i = statePath_deps.begin(); i != statePath_deps.end(); ++i) - sortedStatePaths.push_back(*i); - sort(sortedStatePaths.begin(), sortedStatePaths.end()); - //Convert the Strings into int's and match them to the sorted statePaths - for (vector::const_iterator i = sortedStatePaths.begin(); i != sortedStatePaths.end(); ++i){ + //Get references pointingg to snapshots_table from revisions_table with root_revision + Strings references; + bool notempty = queryStrings(txn, revisions_table, key, references); + + if(!notempty) + throw Error(format("Root revision '%1%' not found of statePath '%2%'") % int2String(root_revision) % statePath); - RevisionNumbers ss_revisions = unpackRevisionNumbers(data.front()); - data.pop_front(); + // + for (Strings::iterator i = references.begin(); i != references.end(); ++i){ + + Path getStatePath; + int getTimestamp; + splitStatePathRevision(*i, getStatePath, getTimestamp); //query state versioined directorys/files vector sortedPaths; - Derivation drv = derivationFromPath(queryStatePathDrvTxn(txn, statePath)); + Derivation drv = derivationFromPath(queryStatePathDrvTxn(txn, getStatePath)); DerivationStateOutputs stateOutputs = drv.stateOutputs; DerivationStateOutputDirs stateOutputDirs = drv.stateOutputDirs; for (DerivationStateOutputDirs::const_iterator j = stateOutputDirs.begin(); j != stateOutputDirs.end(); ++j){ string thisdir = (j->second).path; - string fullstatedir = statePath + "/" + thisdir; + string fullstatedir = getStatePath + "/" + thisdir; if(thisdir == "/") //exception for the root dir fullstatedir = statePath + "/"; sortedPaths.push_back(fullstatedir); } sort(sortedPaths.begin(), sortedPaths.end()); //sort - - //link - map revisions_ss; - for (vector::const_iterator j = sortedPaths.begin(); j != sortedPaths.end(); ++j){ - revisions_ss[*j] = ss_revisions.front(); - ss_revisions.pop_front(); + + Strings snapshots_s; + Snapshots snapshots; + queryStrings(txn, snapshots_table, *i, snapshots_s); + int counter=0; + for (Strings::iterator j = snapshots_s.begin(); j != snapshots_s.end(); ++j){ + + unsigned int revision; + bool succeed = string2UnsignedInt(*j, revision); + if(!succeed) + throw Error(format("Malformed epoch (snapshot timestamp) value of path '%1%'") % statePath); + + snapshots[sortedPaths.at(counter)] = revision; + counter++; } - - revisions[*i] = revisions_ss; + + revisions[getStatePath] = snapshots; } - if(!notempty) - throw Error(format("Root revision '%1%' not found of statePath '%2%'") % int2String(root_revision) % statePath); return notempty; } //TODO include comments into revisions? -bool Database::queryAvailableStateRevisions(const Transaction & txn, TableId table, +bool Database::queryAvailableStateRevisions(const Transaction & txn, TableId revisions_table, const Path & statePath, RevisionNumbers & revisions) { Strings keys; - enumTable(txn, table, keys); //get all revisions + enumTable(txn, revisions_table, keys); //get all revisions for (Strings::const_iterator i = keys.begin(); i != keys.end(); ++i) { - if((*i).substr(0, statePath.length()) != statePath || (*i).length() == statePath.length()) //dont check the new-revision key or other keys + if((*i).substr(0, statePath.length()) != statePath || (*i).length() == statePath.length()) //dont check the new-revision key or other keys continue; Path getStatePath; diff --git a/src/libstore/db.hh b/src/libstore/db.hh index 6f9f08b5e..5f8e9274b 100644 --- a/src/libstore/db.hh +++ b/src/libstore/db.hh @@ -115,15 +115,15 @@ public: const Path & statePath, Strings & referrers, int revision = -1); /* Set the revision number of the statePath and the revision numbers of all state paths in the references closure */ - void setStateRevisions(const Transaction & txn, TableId table, - const Path & statePath, const RevisionNumbersSet & revisions); + void setStateRevisions(const Transaction & txn, TableId revisions_table, TableId snapshots_table, + const Path & statePath, const RevisionClosure & revisions); /* Returns all the revision numbers of the state references closure of the given state path */ - bool queryStateRevisions(const Transaction & txn, TableId table, const PathSet statePath_deps, - const Path & statePath, RevisionNumbersSet & revisions, int root_revision = -1); + bool queryStateRevisions(const Transaction & txn, TableId revisions_table, TableId snapshots_table, + const Path & statePath, RevisionClosure & revisions, int root_revision = -1); /* Returns all available revision numbers of the given state path */ - bool queryAvailableStateRevisions(const Transaction & txn, TableId table, + bool queryAvailableStateRevisions(const Transaction & txn, TableId revisions_table, const Path & statePath, RevisionNumbers & revisions); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 95753e50e..de6a0bc8b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -124,13 +124,20 @@ static TableId dbStateCounters = 0; */ static TableId dbStateInfo = 0; -/* dbStateRevisions :: StatePath -> RevisionNumbers +/* dbStateRevisions :: StatePath -> [StatePath] - This table lists the statepaths + recursive (indirect) references and the revision numbers of their repositorys + This table lists the ............... */ static TableId dbStateRevisions = 0; +/* dbStateSnapshots :: StatePath -> RevisionNumbers + + This table lists the ............... + +*/ +static TableId dbStateSnapshots = 0; + /* dbSharedState :: Path -> Path * * Lists all paths that are shared with other paths @@ -211,6 +218,7 @@ LocalStore::LocalStore(bool reserveSpace) dbStateComponentReferrers = nixDB.openTable("referrers_s_c", true); dbStateStateReferrers = nixDB.openTable("referrers_s_s", true); dbStateRevisions = nixDB.openTable("staterevisions"); + dbStateSnapshots = nixDB.openTable("stateSnapshots"); dbSharedState = nixDB.openTable("sharedState"); dbSolidStateReferences = nixDB.openTable("references_solid_c_s"); /* The contents of this table is included in references_c_s */ @@ -1661,26 +1669,22 @@ void queryAllValidPaths(const Transaction & txn, PathSet & allComponentPaths, Pa } -void setStateRevisionsTxn(const Transaction & txn, const Path & statePath, const RevisionNumbersSet & revisions) +void setStateRevisionsTxn(const Transaction & txn, const Path & statePath, const RevisionClosure & revisions) { - nixDB.setStateRevisions(txn, dbStateRevisions, statePath, revisions); + nixDB.setStateRevisions(txn, dbStateRevisions, dbStateSnapshots, statePath, revisions); } -void LocalStore::setStateRevisions(const Path & statePath, const RevisionNumbersSet & revisions) +void LocalStore::setStateRevisions(const Path & statePath, const RevisionClosure & revisions) { nix::setStateRevisionsTxn(noTxn, statePath, revisions); } -bool queryStateRevisionsTxn(const Transaction & txn, const Path & statePath, RevisionNumbersSet & revisions, const int revision) +bool queryStateRevisionsTxn(const Transaction & txn, const Path & statePath, RevisionClosure & revisions, const int revision) { - PathSet statePaths; - storePathRequisites(statePath, false, statePaths, false, true, revision); //Get all current state dependencies - statePaths.insert(statePath); //also insert the root statePath - - return nixDB.queryStateRevisions(txn, dbStateRevisions, statePaths, statePath, revisions, revision); + return nixDB.queryStateRevisions(txn, dbStateRevisions, dbStateSnapshots, statePath, revisions, revision); } -bool LocalStore::queryStateRevisions(const Path & statePath, RevisionNumbersSet & revisions, const int revision) +bool LocalStore::queryStateRevisions(const Path & statePath, RevisionClosure & revisions, const int revision) { return nix::queryStateRevisionsTxn(noTxn, statePath, revisions, revision); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index da93b9044..4e9574259 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -100,9 +100,9 @@ public: void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState, const int revision); - void setStateRevisions(const Path & statePath, const RevisionNumbersSet & revisions); + void setStateRevisions(const Path & statePath, const RevisionClosure & revisions); - bool queryStateRevisions(const Path & statePath, RevisionNumbersSet & revisions, const int revision); + bool queryStateRevisions(const Path & statePath, RevisionClosure & revisions, const int revision); bool queryAvailableStateRevisions(const Path & statePath, RevisionNumbers & revisions); @@ -233,7 +233,7 @@ void queryStateReferrersTxn(const Transaction & txn, const Path & storePath, Pat Path queryStatePathDrvTxn(const Transaction & txn, const Path & statePath); void storePathRequisitesTxn(const Transaction & txn, const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState, const int revision); -void setStateRevisionsTxn(const Transaction & txn, const Path & statePath, const RevisionNumbersSet & revisions); +void setStateRevisionsTxn(const Transaction & txn, const Path & statePath, const RevisionClosure & revisions); bool isValidPathTxn(const Transaction & txn, const Path & path); bool isValidStatePathTxn(const Transaction & txn, const Path & path); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9e2e9955e..dd19a7069 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -469,13 +469,13 @@ void RemoteStore::storePathRequisites(const Path & storeOrstatePath, const bool } //TODO -void RemoteStore::setStateRevisions(const Path & statePath, const RevisionNumbersSet & revisions) +void RemoteStore::setStateRevisions(const Path & statePath, const RevisionClosure & revisions) { } //TODO -bool RemoteStore::queryStateRevisions(const Path & statePath, RevisionNumbersSet & revisions, const int revision) +bool RemoteStore::queryStateRevisions(const Path & statePath, RevisionClosure & revisions, const int revision) { return false; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 6fc97864f..94f4441e5 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -88,9 +88,9 @@ public: void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState, const int revision); - void setStateRevisions(const Path & statePath, const RevisionNumbersSet & revisions); + void setStateRevisions(const Path & statePath, const RevisionClosure & revisions); - bool queryStateRevisions(const Path & statePath, RevisionNumbersSet & revisions, const int revision); + bool queryStateRevisions(const Path & statePath, RevisionClosure & revisions, const int revision); bool queryAvailableStateRevisions(const Path & statePath, RevisionNumbers & revisions); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 6ac58b56e..2f985648c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -151,29 +151,6 @@ void checkStatePath(const Derivation & drv) Error(format("The statepath from the Derivation does not match the recalculated statepath, are u trying to spoof the statepath?")); } -Path getStateReposPath(const string & type, const Path statePath) -{ - //This is a little trick: we could use the same hash as the statepath, but we change it so the repository also gets a unique scannable hash - Hash hash = hashString(htSHA256, statePath); - - //Extract suffix and stateIdentifier from statePath - int pos = statePath.find_first_of("-"); - string suffix = statePath.substr(pos, statePath.length()); - - /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":sha256:" + printHash(hash) + ":" - + nixStoreState + ":" + suffix; - - checkStoreName(suffix); - - Path path = nixStoreStateRepos + "/" - + printHash32(compressHash(hashString(htSHA256, s), 20)) - + suffix + "/"; - - return path; -} - - Path makeFixedOutputPath(bool recursive, string hashAlgo, Hash hash, string name) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0d019a667..1cb06bc17 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -220,10 +220,10 @@ public: virtual void storePathRequisites(const Path & storeOrstatePath, const bool includeOutputs, PathSet & paths, const bool & withComponents, const bool & withState, const int revision) = 0; /* TODO */ - virtual void setStateRevisions(const Path & statePath, const RevisionNumbersSet & revisions) = 0; + virtual void setStateRevisions(const Path & statePath, const RevisionClosure & revisions) = 0; /* TODO */ - virtual bool queryStateRevisions(const Path & statePath, RevisionNumbersSet & revisions, const int revision) = 0; + virtual bool queryStateRevisions(const Path & statePath, RevisionClosure & revisions, const int revision) = 0; /* TODO */ virtual bool queryAvailableStateRevisions(const Path & statePath, RevisionNumbers & revisions) = 0; @@ -265,9 +265,6 @@ Path makeStatePath(const string & componentHash, const string & suffix, const st /* TODO ... */ void checkStatePath(const Derivation & drv); -/* Calculates a unique store state repos path */ -Path getStateReposPath(const string & type, const Path statePath); - /* This is the preparatory part of addToStore() and addToStoreFixed(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the diff --git a/src/libstore/store-state.cc b/src/libstore/store-state.cc index dbe590553..c7f051145 100644 --- a/src/libstore/store-state.cc +++ b/src/libstore/store-state.cc @@ -57,10 +57,7 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De Path fullstatedir = stateDir + "/" + thisdir; - Strings p_args; - p_args.push_back("-p"); - p_args.push_back(fullstatedir); - runProgram_AndPrintOutput("mkdir", true, p_args, "mkdir"); + ensureDirExists(fullstatedir); if(d.type == "interval"){ intervalPaths.insert(fullstatedir); @@ -72,10 +69,8 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De store->setStatePathsInterval(intervalPaths, empty, true); } -void commitStatePathTxn(const Transaction & txn, const Path & statePath) +Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath) { - //TODO: include code from: updateRevisionsRecursivelyTxn(txn, root_statePath); - if(!isValidStatePathTxn(txn, statePath)) throw Error(format("path `%1%' is not a valid state path") % statePath); @@ -100,6 +95,8 @@ void commitStatePathTxn(const Transaction & txn, const Path & statePath) } vector intervals = store->getStatePathsInterval(intervalPaths); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! txn + Snapshots revisions_list; + int intervalAt=0; for (DerivationStateOutputDirs::const_iterator i = stateOutputDirs.begin(); i != stateOutputDirs.end(); ++i){ DerivationStateOutputDir d = i->second; @@ -122,92 +119,31 @@ void commitStatePathTxn(const Transaction & txn, const Path & statePath) intervals[intervalAt] = interval_counter + 1; intervalAt++; - if(interval_counter % interval != 0){ return; } + if(interval_counter % interval != 0){ continue; } } else if(d.type == "full"){ } - else if(d.type == "manual"){ return; } //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + else if(d.type == "manual"){ continue; } //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! else throw Error(format("Type '%1%' is not handled in nix-state") % d.type); //We got here so we need to commit - unsigned int epoch_time; + unsigned int revision_number; if(pathExists(fullstatedir) || FileExist(fullstatedir)){ - epoch_time = take_snapshot(fullstatedir); - printMsg(lvlError, format("Snapshotted '%1%' with id '%2%'") % fullstatedir % epoch_time); + revision_number = take_snapshot(fullstatedir); + printMsg(lvlError, format("Snapshotted '%1%' with id '%2%'") % fullstatedir % revision_number); } else - { - //TODO !!!!!!!!!!!!!! - } + revision_number = 0; //deleted, so we assign 0 to indicate that - //Put in database !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - //TODO + revisions_list[fullstatedir] = revision_number; } + return revisions_list; + //Update the intervals again //setStatePathsIntervalTxn(txn, intervalPaths, intervals); //TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!! uncomment } -/* - * This function takes all state requisites (references) and revision numbers and stores them ... - */ -/* -void updateRevisionsRecursivelyTxn(const Transaction & txn, const Path & statePath) -{ - //Save all revisions for the call to - RevisionNumbersSet rivisionMapping; - - PathSet statePaths; - storePathRequisitesTxn(txn, statePath, false, statePaths, false, true, -1); //Get all current state dependencies - - //Add own statePath (may already be in there, but its a set, so no doubles) - statePaths.insert(statePath); - - //Sort - vector sortedStatePaths; - for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) - sortedStatePaths.push_back(*i); - sort(sortedStatePaths.begin(), sortedStatePaths.end()); - - //call scanForAllReferences again on all newly found statePaths - for (vector::const_iterator i = sortedStatePaths.begin(); i != sortedStatePaths.end(); ++i) - rivisionMapping[*i] = readRevisionNumber(*i); - - //Store the revision numbers in the database for this statePath with revision number - setStateRevisionsTxn(txn, statePath, rivisionMapping); -} - -int readRevisionNumber(Path statePath) -{ - string svnbin = nixSVNPath + "/svn"; - RevisionNumbers revisions; - - string repos = getStateReposPath("stateOutput:staterepospath", statePath); //this is a copy from store-state.cc - - //TODO Check if the .svn exists, it might be deleted, then we dont have to remember the state revision (set -1) - - Strings p_args; - p_args.push_back(svnbin); - p_args.push_back("file://" + repos); - string output = runProgram(nixLibexecDir + "/nix/nix-readrevisions.sh", true, p_args); //run - - int pos = output.find("\n",0); //remove trailing \n - output.erase(pos,1); - - int revision; - bool succeed = string2Int(output, revision); - if(!succeed) - throw Error(format("Cannot read revision number of path '%1%'") % repos); - - return revision; -} -*/ - -// string s = "/media/ext3cow/cca/"; -// unsigned int i = take_snapshot(s); -// printMsg(lvlError, format("SS: '%1%'") % i); - - } diff --git a/src/libstore/store-state.hh b/src/libstore/store-state.hh index 65ad2fd41..aacaf4eca 100644 --- a/src/libstore/store-state.hh +++ b/src/libstore/store-state.hh @@ -11,7 +11,7 @@ namespace nix { void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const DerivationStateOutputs & stateOutputs); /* TODO */ -void commitStatePathTxn(const Transaction & txn, const Path & statePath); +Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath); /* TODO */ //void updateRevisionsRecursivelyTxn(const Transaction & txn, const Path & statePath); diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 40c8cf87a..7fac9238a 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -59,8 +59,9 @@ typedef list Paths; typedef set PathSet; //state types -typedef list RevisionNumbers; //the Strings (list) of StateReferences and this list are connected by position -typedef map > RevisionNumbersSet; //We include to the paths to sort on +typedef list RevisionNumbers; //the Strings (list) of StateReferences and this list are connected by position +typedef map Snapshots; //Automatically sorted on Path :) +typedef map RevisionClosure; typedef map StateReferences; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index fc4138717..deea519a3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1039,6 +1039,20 @@ bool string2Int(const string & s, int & n) return str && str.get() == EOF; } +string unsignedInt2String(unsigned int n) +{ + std::ostringstream str; + str << n; + return str.str(); +} + +bool string2UnsignedInt(const string & s, unsigned int & n) +{ + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; +} + string bool2string(const bool b) { if(b == true) @@ -1126,11 +1140,11 @@ void executeShellCommand(const string & command) } } -string time_t2string(const time_t & t) +int getTimeStamp() { - int i = t; - string s = int2String(i); - return s; + const time_t now = time(0); + int i = now; + return i; } //TODO Does this work on windows? @@ -1218,5 +1232,12 @@ void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, Pat } } +void ensureDirExists(const Path & path) +{ + Strings p_args; + p_args.push_back("-p"); + p_args.push_back(path); + runProgram_AndPrintOutput("mkdir", true, p_args, "mkdir"); //TODO ensurePath +} } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3381c164d..41868757a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -280,6 +280,10 @@ bool statusOk(int status); string int2String(int n); bool string2Int(const string & s, int & n); +/* */ +bool string2UnsignedInt(const string & s, unsigned int & n); +string unsignedInt2String(unsigned int n); + /* Parse a bool to a string and back */ string bool2string(const bool b); bool string2bool(const string & s); @@ -297,8 +301,7 @@ void executeShellCommand(const string & command); // void runProgram_AndPrintOutput(Path program, bool searchPath, const Strings & args, const string outputPrefix); -//Convert time_t to a string -string time_t2string(const time_t & t); +int getTimeStamp(); bool FileExist(const string FileName); @@ -312,9 +315,8 @@ PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2); /* TODO */ void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths); -string packRevisionNumbers(const RevisionNumbers & revs); - -RevisionNumbers unpackRevisionNumbers(const string & packed); + +void ensureDirExists(const Path & path); } diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index da38afe54..d55394270 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -1,5 +1,6 @@ #include #include +#include #include "globals.hh" #include "misc.hh" @@ -158,7 +159,7 @@ static void opShowStatePath(Strings opFlags, Strings opArgs) } //Prints the root path that contains the repoisitorys of the state of a component - indetiefier combination -static void opShowStateReposPath(Strings opFlags, Strings opArgs) +static void opShowStatePathAtRevision(Strings opFlags, Strings opArgs) { Path componentPath; Path statePath; @@ -171,10 +172,9 @@ static void opShowStateReposPath(Strings opFlags, Strings opArgs) if(!isStateComponent) throw UsageError(format("This path '%1%' is not a state-component path") % componentPath); - //Get the a repository for this state location - string repos = getStateReposPath("stateOutput:staterepospath", statePath); //this is a copy from store-state.cc - - printMsg(lvlError, format("%1%") % repos); + //TODO + + //printMsg(lvlError, format("%1%") % repos); } @@ -236,6 +236,7 @@ static void revertToRevision(Strings opFlags, Strings opArgs) bool isStateComponent; string program_args; Derivation drv = getDerivation_andCheckArgs(opFlags, opArgs, componentPath, statePath, binary, derivationPath, isStateComponent, program_args); + DerivationStateOutputDirs stateOutputDirs = drv.stateOutputDirs; bool recursive = true; //TODO !!!!!!!!!!!!!!!!! @@ -246,15 +247,71 @@ static void revertToRevision(Strings opFlags, Strings opArgs) statePaths.insert(derivationPath); //Insert direct state path //Get the revisions recursively to also roll them back - RevisionNumbersSet getRivisions; + RevisionClosure getRivisions; bool b = store->queryStateRevisions(statePath, getRivisions, revision_arg); //Revert each statePath in the list - for (RevisionNumbersSet::iterator i = getRivisions.begin(); i != getRivisions.end(); ++i){ + for (RevisionClosure::iterator i = getRivisions.begin(); i != getRivisions.end(); ++i){ Path statePath = (*i).first; - map revisioned_paths = (*i).second; + Snapshots revisioned_paths = (*i).second; - //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + //TODO Sort snapshots??? eg first restore root, then the subdirs?? + + for (Snapshots::iterator j = revisioned_paths.begin(); j != revisioned_paths.end(); ++j){ + Path revertPathOrFile = (*j).first; + unsigned int epoch = (*j).second; + + //printMsg(lvlError, format("MAYBE '%1%'") % revertPathOrFile); + + //Look up the type from the drv with for the current snapshotted path + Path statePath_postfix = revertPathOrFile.substr(nixStoreState.length() + 1, revertPathOrFile.length() - nixStoreState.length()); + statePath_postfix = statePath_postfix.substr(statePath_postfix.find_first_of("/") + 1, statePath_postfix.length()); + if(statePath_postfix == "") + statePath_postfix = "/"; + string type = stateOutputDirs.at(statePath_postfix).type; + if(type == "none") + continue; + + //Now that were still here, we need to copy the state from the previous version back + Strings p_args; + p_args.push_back("-c"); //we use the shell to execute the cp command becuase the shell expands the '*' + string cpcommand = "cp -R"; + if(revertPathOrFile.substr(revertPathOrFile.length() -1 , revertPathOrFile.length()) == "/"){ //is dir + cpcommand += " " + (revertPathOrFile.substr(0, revertPathOrFile.length() -1) + "@" + unsignedInt2String(epoch) + "/*"); + + //clean all contents of the folder first (so were sure the path is clean) + if(pathExists(revertPathOrFile)) + deletePath(revertPathOrFile); + + //If path was not deleted in the previous version, we need to make sure it exists or cp will fail + if(epoch == 0) + continue; + else + ensureDirExists(revertPathOrFile); + } + else{ //is file + cpcommand += " " + (revertPathOrFile + "@" + unsignedInt2String(epoch)); + + if(epoch == 0){ + //delete file + if(FileExist(revertPathOrFile)) + deletePath(revertPathOrFile); //we only delete if the cp doesnt overwrite it below + continue; + } + } + + printMsg(lvlError, format("Reverting '%1%'") % revertPathOrFile); + + cpcommand += " " + revertPathOrFile; + p_args.push_back(cpcommand); + + //for (Strings::iterator h = p_args.begin(); h != p_args.end(); ++h) + // printMsg(lvlError, format("SH ARGS '%1%'") % *h); + + runProgram_AndPrintOutput("sh", true, p_args, "sh-cp"); //TODO does this work on windows? + } + + printMsg(lvlError, format("Reverted state of '%1%' to revision '%2%'") % statePath % revision_arg); } } @@ -357,7 +414,8 @@ void scanAndUpdateAllReferencesRecusivelyTxn(const Transaction & txn, const Path //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! int revision = 0; - //Get last revision number from DB !!!!!!!!!! + //Get last revision number from DB !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + //TODO first snapshot all paths....? //Scan, update, call recursively PathSet newFoundComponentReferences; @@ -408,7 +466,7 @@ static void opRunComponent(Strings opFlags, Strings opArgs) //******************* Run **************************** if(!only_commit) - executeShellCommand(root_componentPath + root_binary + " " + root_program_args); //more efficient way needed ??? + executeShellCommand(root_componentPath + root_binary + " " + root_program_args); //******************* With everything in place, we call the commit script on all statePaths (in)directly referenced ********************** @@ -419,16 +477,20 @@ static void opRunComponent(Strings opFlags, Strings opArgs) scanAndUpdateAllReferencesRecusivelyTxn(txn, root_statePath); //Commit all statePaths + RevisionClosure rivisionMapping; for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) //TODO first commit own state path? - commitStatePathTxn(txn, *i); + rivisionMapping[*i] = commitStatePathTxn(txn, *i); + + //Save new revisions + setStateRevisionsTxn(txn, root_statePath, rivisionMapping); //Commit transaction //txn.commit(); //Debugging - RevisionNumbersSet getRivisions; + RevisionClosure getRivisions; bool b = store->queryStateRevisions(root_statePath, getRivisions, -1); - for (RevisionNumbersSet::iterator i = getRivisions.begin(); i != getRivisions.end(); ++i){ + for (RevisionClosure::iterator i = getRivisions.begin(); i != getRivisions.end(); ++i){ //printMsg(lvlError, format("State %1% has revision %2%") % (*i).first % int2String((*i).second)); } @@ -537,10 +599,19 @@ void run(Strings args) for (Strings::iterator i = strings2.begin(); i != strings2.end(); ++i) printMsg(lvlError, format("UN '%1%'") % *i); - */ - //updateRevisionNumbers("/nix/state/xf582zrz6xl677llr07rvskgsi3dli1d-hellohardcodedstateworld-dep1-1.0-test"); //return; + + //printMsg(lvlError, format("NOW: '%1%'") % getTimeStamp()); + + //auto sort + map test; + test["q"] = "324"; + test["c"] = "3241"; + test["a"] = "a"; + for (map::const_iterator j = test.begin(); j != test.end(); ++j) + printMsg(lvlError, format("KEY: '%1%'") % (*j).first); + */ /* test */ @@ -557,8 +628,6 @@ void run(Strings args) } else if (arg == "--showstatepath") op = opShowStatePath; - else if (arg == "--showstatereposrootpath") - op = opShowStateReposPath; else if (arg == "--showderivations") op = opShowDerivations; else if (arg == "--showrevisions") @@ -571,10 +640,9 @@ void run(Strings args) } /* - - --commit - --run-without-commit + + --show-revision-path=.... --showrevisions