From 4fb9070fbd81d61b596ef9a7b2043c1be7ac1421 Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Fri, 3 Aug 2007 10:28:58 +0000 Subject: [PATCH] before removing referrer code --- src/libstore/db.cc | 89 +++++++++++++++++++++------ src/libstore/db.hh | 8 ++- src/libstore/local-store.cc | 117 ++++++++++++++++++++++++------------ src/nix-state/nix-state.cc | 14 ++++- 4 files changed, 168 insertions(+), 60 deletions(-) diff --git a/src/libstore/db.cc b/src/libstore/db.cc index 54e4d1850..038f58bfe 100644 --- a/src/libstore/db.cc +++ b/src/libstore/db.cc @@ -449,13 +449,13 @@ void Database::enumTable(const Transaction & txn, TableId table, /* State specific db functions */ -Path Database::makeStatePathRevision(const Path & statePath, const int revision) +Path Database::mergeToDBKey(const Path & statePath, const int revision) { string prefix = "-KEY-"; return statePath + prefix + int2String(revision); } -void Database::splitStatePathRevision(const Path & revisionedStatePath, Path & statePath, int & revision) +void Database::splitDBKey(const Path & revisionedStatePath, Path & statePath, int & revision) { string prefix = "-KEY-"; @@ -503,7 +503,7 @@ bool Database::lookupHighestRevivison(const Strings & keys, const Path & statePa //printMsg(lvlError, format("'%1%' - '%2%'") % *i % statePath); Path getStatePath; int getRevision; - splitStatePathRevision(*i, getStatePath, getRevision); + splitDBKey(*i, getStatePath, getRevision); if(getRevision > highestRev){ if(lowerthan != -1){ @@ -518,19 +518,19 @@ bool Database::lookupHighestRevivison(const Strings & keys, const Path & statePa if(highestRev == -1) //no records found return false; - key = makeStatePathRevision(statePath, highestRev); //final key that matches revision + statePath + key = mergeToDBKey(statePath, highestRev); //final key that matches revision + statePath return true; } bool Database::revisionToTimeStamp(const Transaction & txn, TableId revisions_table, const Path & statePath, const int revision, int & timestamp) { - string key = makeStatePathRevision(statePath, revision); + string key = mergeToDBKey(statePath, revision); Strings references; bool notempty = queryStrings(txn, revisions_table, key, references); if(notempty){ Path empty; - splitStatePathRevision(*(references.begin()), empty, timestamp); //extract the timestamp + splitDBKey(*(references.begin()), empty, timestamp); //extract the timestamp //printMsg(lvlError, format("PRINT '%1%'") % timestamp); return true; } @@ -561,10 +561,15 @@ void Database::setStateReferences(const Transaction & txn, TableId references_ta printMsg(lvlError, format("Warning: The timestamp '%1%' already exists for set-references/referrers of path '%2%' with db '%3%'") % timestamp % statePath % references_table); //Create the key - string key = makeStatePathRevision(statePath, timestamp); + string key = mergeToDBKey(statePath, timestamp); + + + printMsg(lvlError, format("Set references '%1%'") % key); + for (Strings::const_iterator i = references.begin(); i != references.end(); ++i) + printMsg(lvlError, format("reference '%1%'") % *i); //Insert - setStrings(txn, references_table, key, references); + setStrings(txn, references_table, key, references, false); //The false makes sure also empty references are set } bool Database::queryStateReferences(const Transaction & txn, TableId references_table, TableId revisions_table, @@ -593,12 +598,12 @@ bool Database::queryStateReferences(const Transaction & txn, TableId references_ } //If a specific key is given: check if this timestamp exists key in the table - key = makeStatePathRevision(statePath, timestamp); + key = mergeToDBKey(statePath, timestamp); bool found = false; for (Strings::const_iterator i = keys.begin(); i != keys.end(); ++i) { if(key == *i){ found = true; - key = makeStatePathRevision(statePath, timestamp); + key = mergeToDBKey(statePath, timestamp); } } @@ -627,6 +632,55 @@ bool Database::queryStateReferrers(const Transaction & txn, TableId referrers_ta return queryStateReferences(txn, referrers_table, revisions_table, statePath, referrers, revision, timestamp); } + //PRIVATE !!!!!! +//Update the referrers of some path because the store_or_statePath got new references +void Database::updateReferredPath(const Transaction & txn, TableId revisions_table, TableId referrers_X_s_table, + const Path & referred_state_or_store_Path, const Path & statePath, const int revision) +{ + + //referred_state_or_store_Path --> statePath + + //you dont know what referred_state_or_store_Path is !!!!!!!!!!!! add a bool !!!!!! + + if(revision != -1){ + //Get timestamp of revision + int timestamp; + bool succeed = revisionToTimeStamp(txn, revisions_table, referred_state_or_store_Path, revision, timestamp); + if(!succeed) + throw Error(format("Couldnt find timestamp for revision '%1%' at update referrers mapping for '%2%'") % revision % referred_state_or_store_Path); + + //remove all dynamic (that have a ts) referrers of that revision first (except if it is the latest, eg. -1) + delPair(txn, referrers_X_s_table, mergeToDBKey(referred_state_or_store_Path, timestamp)); + + } + + //now get the referrers at timestamp (or just the latest if revision is -1) + PathSet referrers; + //TODO + + //if( ! state) + //getReferrers............... + //dbStateComponentReferrers + //else + //getStateReferrers............... + //dbStateStateReferrers + + + //string kkkk = nixDB.mergeToDBKey(store_or_statePath, revision); + + //Add store_or_statePath in .... if nessacary (if in references) + //TODO + //referredPath - TS --> store_or_statePath ADD + + //Set Remove store_or_statePath in .... if nessacary (if not in references) + //TODO + //referredPath - TS --> store_or_statePath DEL + + + //set the new referrers of revision (-1 insert as a new timestamp) + //nixDB.setStateReferrers(txn, table, dbStateRevisions, referredPath, ........, revision); +} + void Database::setStateRevisions(const Transaction & txn, TableId revisions_table, TableId snapshots_table, const RevisionClosure & revisions) { @@ -634,7 +688,7 @@ void Database::setStateRevisions(const Transaction & txn, TableId revisions_tabl //Insert all ss_epochs into snapshots_table with the current ts. for (RevisionClosure::const_iterator i = revisions.begin(); i != revisions.end(); ++i){ - string key = makeStatePathRevision((*i).first, ts); + string key = mergeToDBKey((*i).first, ts); Strings data; //the map<> takes care of the sorting on the Path for (Snapshots::const_iterator j = (*i).second.begin(); j != (*i).second.end(); ++j) @@ -647,7 +701,7 @@ void Database::setStateRevisions(const Transaction & txn, TableId revisions_tabl Path statePath = (*i).first; int revision = getNewRevisionNumber(txn, revisions_table, statePath); //get a new revision number - string key = makeStatePathRevision(statePath, revision); + string key = mergeToDBKey(statePath, revision); //get all its requisites PathSet statePath_references; @@ -657,8 +711,9 @@ void Database::setStateRevisions(const Transaction & txn, TableId revisions_tabl //save in db Strings data; for (PathSet::const_iterator j = statePath_references.begin(); j != statePath_references.end(); ++j) - data.push_back(makeStatePathRevision(*j, ts)); - setStrings(txn, revisions_table, key, data); + data.push_back(mergeToDBKey(*j, ts)); + + setStrings(txn, revisions_table, key, data, false); //The false makes sure also empty revisions are set } } @@ -675,7 +730,7 @@ bool Database::queryStateRevisions(const Transaction & txn, TableId revisions_ta return false; } else - key = makeStatePathRevision(statePath, root_revision); + key = mergeToDBKey(statePath, root_revision); //Get references pointing to snapshots_table from revisions_table with root_revision Strings statePaths; @@ -689,7 +744,7 @@ bool Database::queryStateRevisions(const Transaction & txn, TableId revisions_ta Path getStatePath; int getTimestamp; - splitStatePathRevision(*i, getStatePath, getTimestamp); + splitDBKey(*i, getStatePath, getTimestamp); //query state versioined directorys/files vector sortedPaths; @@ -742,7 +797,7 @@ bool Database::queryAvailableStateRevisions(const Transaction & txn, TableId rev Path getStatePath; int getRevision; - splitStatePathRevision(*i, getStatePath, getRevision); + splitDBKey(*i, getStatePath, getRevision); revisions.push_back(getRevision); } diff --git a/src/libstore/db.hh b/src/libstore/db.hh index 89d659036..9355ef435 100644 --- a/src/libstore/db.hh +++ b/src/libstore/db.hh @@ -100,10 +100,10 @@ public: Strings & keys, const string & keyPrefix = ""); /* TODO */ - Path makeStatePathRevision(const Path & statePath, const int revision); + Path mergeToDBKey(const Path & statePath, const int revision); /* TODO */ - void splitStatePathRevision(const Path & revisionedStatePath, Path & statePath, int & revision); + void splitDBKey(const Path & revisionedStatePath, Path & statePath, int & revision); /* Set the stateReferences for a specific revision (and older until the next higher revision number in the table) */ void setStateReferences(const Transaction & txn, TableId references_table, TableId revisions_table, @@ -120,6 +120,10 @@ public: /* Returns the referrers for a specific revision (and older until the next higher revision number in the table) */ bool queryStateReferrers(const Transaction & txn, TableId referrers_table, TableId revisions_table, const Path & statePath, Strings & referrers, int revision = -1, int timestamp = -1); + + /* TODO */ + void updateReferredPath(const Transaction & txn, TableId revisions_table, TableId referrers_X_s_table, + const Path & referred_state_or_store_Path, const Path & statePath, const int revision); /* Set the revision number of the statePath and the revision numbers of all state paths in the references closure */ void setStateRevisions(const Transaction & txn, TableId revisions_table, TableId snapshots_table, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d36806df1..cd1a80f4b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -418,6 +418,8 @@ static string stripPrefix(const string & prefix, const string & s) return string(s, prefix.size() + 1); } + + void setReferences(const Transaction & txn, const Path & store_or_statePath, const PathSet & references, const PathSet & stateReferences, const int revision) { @@ -445,13 +447,13 @@ void setReferences(const Transaction & txn, const Path & store_or_statePath, PathSet oldReferences = PathSet(oldReferences_c_c.begin(), oldReferences_c_c.end()); PathSet oldStateReferences = PathSet(oldReferences_c_s.begin(), oldReferences_c_s.end()); - if (oldReferences == references && oldStateReferences == stateReferences) return; + if (oldReferences == references && oldStateReferences == stateReferences) return; //watch out we way need to set the referrers.... (at a ts) nixDB.setStrings(txn, dbComponentComponentReferences, store_or_statePath, Paths(references.begin(), references.end())); nixDB.setStrings(txn, dbComponentStateReferences, store_or_statePath, Paths(stateReferences.begin(), stateReferences.end())); - //Handle referrers ...... - + //Handle referrers like the old way, because references (and so referrers) referrers never change (So these do not have a timestamp) + //dbComponentComponentReferrers //dbComponentStateReferrers /* @@ -477,53 +479,27 @@ void setReferences(const Transaction & txn, const Path & store_or_statePath, PathSet oldReferences = PathSet(oldStateReferences_s_c.begin(), oldStateReferences_s_c.end()); PathSet oldStateReferences = PathSet(oldStateReferences_s_s.begin(), oldStateReferences_s_s.end()); - //if (oldReferences == references && oldStateReferences == stateReferences) return; //TODO Turn on ???????????? //set the references of revision (-1 insert as a new timestamp) nixDB.setStateReferences(txn, dbStateComponentReferences, dbStateRevisions, store_or_statePath, Paths(references.begin(), references.end()), revision); nixDB.setStateReferences(txn, dbStateStateReferences, dbStateRevisions, store_or_statePath, Paths(stateReferences.begin(), stateReferences.end()), revision); - //REFERRERS //for all references old and new, state and component - PathSet all = pathSets_union(pathSets_union(references, oldReferences), pathSets_union(oldStateReferences, stateReferences)); //TODO SPLIT in state and non state - for (PathSet::const_iterator i = all.begin(); i != all.end(); ++i){ - - Path referredPath = *i; //Path referred to by the statePath - - //query the refererrers of revision (-1 is query the latest refererrers) - Paths referrers_s_c; - Paths referrers_s_s; - nixDB.queryStateReferrers(txn, dbStateComponentReferrers, dbStateRevisions, referredPath, referrers_s_c, revision); //may contain other referred by paths - nixDB.queryStateReferrers(txn, dbStateStateReferrers, dbStateRevisions, referredPath, referrers_s_s, revision); - PathSet oldReferrers = PathSet(referrers_s_c.begin(), referrers_s_c.end()); - PathSet oldStateReferrers = PathSet(referrers_s_s.begin(), referrers_s_s.end()); - - - //Add store_or_statePath in if nessacary (if in references) - //TODO - - //Remove store_or_statePath if nessacary (if not in references) - //TODO - - //set the new referrers of revision (-1 insert as a new timestamp) - //nixDB.setStateReferrers(txn, dbStateComponentReferences, dbStateRevisions, referredPath, ........, revision); - //nixDB.setStateReferrers(txn, dbStateStateReferences, dbStateRevisions, referredPath, ......, revision); - - } + PathSet all_references = pathSets_union(references, oldReferences); + PathSet all_stateReferences = pathSets_union(oldStateReferences, stateReferences); + + for (PathSet::const_iterator i = all_references.begin(); i != all_references.end(); ++i) + nixDB.updateReferredPath(txn, dbStateRevisions, dbComponentStateReferrers, *i, store_or_statePath, revision); + + for (PathSet::const_iterator i = all_stateReferences.begin(); i != all_stateReferences.end(); ++i) + nixDB.updateReferredPath(txn, dbStateRevisions, dbStateStateReferrers, *i, store_or_statePath, revision); + } else throw Error(format("Path '%1%' is not a valid component or state path") % store_or_statePath); - //store_or_statePath must be postfixed in case it is a statePath - Path store_or_statePath2 = store_or_statePath; - if(isRealisableStatePath(txn, store_or_statePath)) - store_or_statePath2 = nixDB.makeStatePathRevision(store_or_statePath, revision); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! THIS OK ??? - //NO ???? SINCE THE KEYS SHOULD BE TIMESTAMPED ?? - //WE NEEED TO inlcude the revision !! - - } void queryReferencesTxn(const Transaction & txn, const Path & store_or_statePath, PathSet & references, const int revision, int timestamp) @@ -585,6 +561,7 @@ void LocalStore::queryAllReferences(const Path & path, PathSet & allReferences, static PathSet getXReferrers(const Transaction & txn, const Path & store_or_statePath, const int revision, const TableId & table) { + /* if(isValidPathTxn(txn, store_or_statePath)){ PathSet referrers; Strings keys; @@ -602,6 +579,70 @@ static PathSet getXReferrers(const Transaction & txn, const Path & store_or_stat } else throw Error(format("Path '%1%' is not a valid component or state path") % store_or_statePath); + */ + + //TODO!!!!!!!!!!!!!!!!!!!! use revision !!!!!!!!!!!!!!!!!!!!! add timestamp ????? + + PathSet referrers; + + if(isValidPathTxn(txn, store_or_statePath) || isValidStatePathTxn(txn, store_or_statePath)){ + + //Lookup its where it points to if its shared + Path store_or_statePath_ns; + if(isValidStatePathTxn(txn, store_or_statePath)) + store_or_statePath_ns = toNonSharedPathTxn(txn, store_or_statePath); + + map possible_referrers; //referrer at timestamp + + Strings keys; + nixDB.enumTable(txn, table, keys); + for (Strings::iterator i = keys.begin(); i != keys.end(); ++i){ + if( (*i).substr(0, store_or_statePath_ns.length()) == store_or_statePath_ns){ + if( (*i).length() == store_or_statePath_ns.length() ) + referrers.insert(*i); //referrers from components + else{ + + //now filter the latest of each value + Path empty; + int timestamp; + nixDB.splitDBKey(*i, empty, timestamp); + + string data; + nixDB.queryString(txn, table, *i, data); + + Path statePath; + int added_or_removed; + nixDB.splitDBKey(data, statePath, added_or_removed); + + if(possible_referrers[statePath] != 0) //if not yet set + if(timestamp > possible_referrers[statePath]) + possible_referrers[statePath] = timestamp; + else + possible_referrers[statePath] = timestamp; + } + } + } + + //Check wheter the latest was added or deleted + for (map::iterator i = possible_referrers.begin(); i != possible_referrers.end(); ++i){ + + string data; + nixDB.queryString(txn, table, nixDB.mergeToDBKey(store_or_statePath_ns, (*i).second), data); + + Path statePath; + int added_or_removed; + nixDB.splitDBKey(data, statePath, added_or_removed); + + if(added_or_removed == 1) //now filter on -0 (removed) or -1 (added) + referrers.insert(statePath); + } + + return referrers; + } + else + throw Error(format("Path '%1%' is not a valid component or state path") % store_or_statePath); + + } static PathSet getReferrers(const Transaction & txn, const Path & store_or_statePath, const int revision) diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index 85ff2e2ab..587bd8868 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -415,8 +415,11 @@ void scanAndUpdateAllReferencesTxn(const Transaction & txn, const Path & statePa printMsg(lvlError, format("Removed state reference found!: '%1%' in state path '%2%'") % (*i) % statePath); //If any changes are detected: register the paths valid with a new revision number - if(diff_references_added.size() != 0 || diff_references_removed.size() != 0 || - diff_state_references_added.size() != 0 || diff_state_references_removed.size() != 0 ) + //if(diff_references_added.size() != 0 || diff_references_removed.size() != 0 || + //diff_state_references_added.size() != 0 || diff_state_references_removed.size() != 0 ) + + //We always set the referernces so we know they were scanned (maybe the same) at a certain time + if(true) { printMsg(lvlError, format("Updating new references for statepath: '%1%'") % statePath); Path drvPath = queryStatePathDrvTxn(txn, statePath); @@ -638,10 +641,15 @@ void run(Strings args) test["a"] = "a"; for (map::const_iterator j = test.begin(); j != test.end(); ++j) printMsg(lvlError, format("KEY: '%1%'") % (*j).first); - printMsg(lvlError, format("NOW: '%1%'") % getTimeStamp()); return; + map test; + test["a"] = 1; + test["b"] = 2; + printMsg(lvlError, format("NOW: '%1%'") % test["q"]); + return; + */ /* test */