diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8529bca81..613915955 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1483,13 +1483,12 @@ void DerivationGoal::startBuilder() tmpDir = createTempDir(); /* Create the state directory where the component can store it's state files place */ - //TODO MOVEEEEEEEEEEE //We only create state dirs when state is enabled and when the dirs need to be created before the installation if(drv.stateOutputs.size() != 0){ /* we check the recalculated state path at build time with the correct user for securiyt */ checkStatePath(drv); - + if(drv.stateOutputs.find("state")->second.getCreateDirsBeforeInstall()) createSubStateDirsTxn(noTxn, drv.stateOutputDirs, drv.stateOutputs); } @@ -1727,7 +1726,6 @@ void DerivationGoal::computeClosure() PathSet state_stateReferences; //all state references in statepath map contentHashes; - //TODO MOVE THIS TO A PLACE THAT ALSO GETS CALLED WHEN WE DONT NEED TO BUILD ANYTHING //We create state dirs only when state is enabled and when the dirs need to be created after the installation if(drv.stateOutputs.size() != 0) if(!drv.stateOutputs.find("state")->second.getCreateDirsBeforeInstall()) @@ -1920,8 +1918,10 @@ void DerivationGoal::computeClosure() setStateRevisionsTxn(txn, rivisionMapping, statePath, "Initial build revision."); //Convert stateInfo from drv to DB format + //And set all interval-ed paths to zero to begin with DerivationStateOutputDirs stateOutputDirs = drv.stateOutputDirs; StateInfos infos; + CommitIntervals intervals; for (DerivationStateOutputDirs::const_reverse_iterator j = stateOutputDirs.rbegin(); j != stateOutputDirs.rend(); ++j){ DerivationStateOutputDir d = j->second; StateInfo si; @@ -1936,7 +1936,12 @@ void DerivationGoal::computeClosure() else si.interval = 0; infos.push_back(si); + + + if(d.type == "interval") + intervals[d.path] = 0; } + setStatePathsIntervalTxn(txn, statePath, intervals); //Set intervals to zero setVersionedStateEntriesTxn(txn, statePath, infos); //register subdirs/files and their types of versioning setStateUserGroupTxn(txn, statePath, queryCallingUsername(), "nixbld", 700); //register the user and group @@ -1945,6 +1950,7 @@ void DerivationGoal::computeClosure() if(sharedState != ""){ shareStateTxn(txn, sharedState, statePath, false); } + //If not shared: create the dir and set the rights else{ ensureStateDir(statePath, queryCallingUsername(), "nixbld", "700"); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e52c6022b..bcc34805e 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -582,22 +582,6 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, allLiveStatePaths = pathSets_union(getSharedWithPathSetRecTxn(noTxn, *i), allLiveStatePaths); } - /* - * Lookup all derivations, of all state paths, because they need to be kept for comitting //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - */ - PathSet allStatePathDerivations; - for (PathSet::iterator i = allLiveStatePaths.begin(); i != allLiveStatePaths.end(); ++i){ - //printMsg(lvlError, format("Live state path `%1%'") % *i); - Path stateDrv = queryStatePathDrvTxn(noTxn, *i); - allStatePathDerivations.insert(stateDrv); - //printMsg(lvlError, format("Live state path drv `%1%'") % stateDrv); - - //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! TODO put back on - if(!store->isValidPath(stateDrv)) - {} // throw Error(format("deriver `%1%' of state component `%2%' in GC is not valid") % stateDrv % *i); - - // computeFSClosure(stateDrv, livePaths, true, true, 0); //TODO .................. should we do this ????????????? - } /* Read the Nix store and state directory's to find all currently existing paths. */ @@ -661,11 +645,6 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, continue; } - if (allStatePathDerivations.find(*i) != allStatePathDerivations.end()) { - debug(format("Keeping statePath derivation `%1%'") % *i); - continue; - } - debug(format("dead path `%1%'") % *i); result.insert(*i); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1525172c1..e4a207c57 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1470,57 +1470,41 @@ void verifyStore(bool checkContents) txn.commit(); } -void setStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePaths, const IntVector & intervals, bool allZero) +void setStatePathsIntervalTxn(const Transaction & txn, const Path & statePath, const CommitIntervals & intervals) { - if(!allZero && statePaths.size() != intervals.size()){ - throw Error("the number of statepaths and intervals must be equal"); - } - - int n=0; - for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) - { - //printMsg(lvlError, format("Set interval of PATH: %1%") % *i); - - int interval=0; - if(!allZero) - interval = intervals.at(n); - - nixDB.setString(txn, dbStateCounters, *i, int2String(interval)); - n++; - } + Strings data; + for (CommitIntervals::const_iterator i = intervals.begin(); i != intervals.end(); ++i) + data.push_back(mergeToDBRevKey((*i).first, (*i).second)); + + nixDB.setStrings(txn, dbStateCounters, statePath, data); } -void LocalStore::setStatePathsInterval(const PathSet & statePaths, const IntVector & intervals, bool allZero) +void LocalStore::setStatePathsInterval(const Path & statePath, const CommitIntervals & intervals) { Transaction txn(nixDB); - nix::setStatePathsIntervalTxn(txn, statePaths, intervals, allZero); + nix::setStatePathsIntervalTxn(txn, statePath, intervals); txn.commit(); } -IntVector getStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePaths) +CommitIntervals getStatePathsIntervalTxn(const Transaction & txn, const Path & statePath) { - string data; - Paths referers; - - IntVector intervals; - for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) + Strings data; + nixDB.queryStrings(txn, dbStateCounters, statePath, data); + CommitIntervals intervals; + for (Strings::iterator i = data.begin(); i != data.end(); ++i) { - nixDB.queryString(txn, dbStateCounters, *i, data); - - //Check if every key returns a value from the db - int n; - if (!string2Int(data, n)) - throw Error(format("Statepath `%1%' has returned no valid interval from the database") % *i); - intervals.push_back(n); + Path p; + unsigned int interval; + splitDBRevKey(*i, p, interval); + intervals[p] = interval; } - return intervals; } -IntVector LocalStore::getStatePathsInterval(const PathSet & statePaths) +CommitIntervals LocalStore::getStatePathsInterval(const Path & statePath) { - return nix::getStatePathsIntervalTxn(noTxn, statePaths); + return nix::getStatePathsIntervalTxn(noTxn, statePath); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7cd63d571..661b9fa96 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -110,9 +110,9 @@ public: ///////////////////////////// - void setStatePathsInterval(const PathSet & statePath, const IntVector & intervals, bool allZero = false); + void setStatePathsInterval(const Path & statePath, const CommitIntervals & intervals); - IntVector getStatePathsInterval(const PathSet & statePaths); + CommitIntervals getStatePathsInterval(const Path & statePath); bool isStateComponent(const Path & storePath); @@ -267,10 +267,11 @@ Path toNonSharedPathTxn(const Transaction & txn, const Path & statePath); PathSet getSharedWithPathSetRecTxn(const Transaction & txn, const Path & statePath); void ensurePathTxn(const Transaction & txn, const Path & path); -IntVector getStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePaths); + +CommitIntervals getStatePathsIntervalTxn(const Transaction & txn, const Path & statePath); +void setStatePathsIntervalTxn(const Transaction & txn, const Path & statePath, const CommitIntervals & intervals); bool queryStateRevisionsTxn(const Transaction & txn, const Path & statePath, RevisionClosure & revisions, RevisionClosureTS & timestamps, const unsigned int revision); -void setStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePath, const IntVector & intervals, bool allZero = false); bool querySharedStateTxn(const Transaction & txn, const Path & statePath, Path & shared_with); void setStateComponentTxn(const Transaction & txn, const Path & storePath); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 81e6b0a4c..045799c17 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -461,22 +461,21 @@ PathSet RemoteStore::queryDerivers(const Path & storePath, const string & identi return readStorePaths(from); //TODO is this ok ?? } -void RemoteStore::setStatePathsInterval(const PathSet & statePaths, const IntVector & intervals, bool allZero) +void RemoteStore::setStatePathsInterval(const Path & statePath, const CommitIntervals & intervals) { writeInt(wopSetStatePathsInterval, to); - writeStringSet(statePaths, to); - writeIntVector(intervals, to); - writeInt(allZero ? 1 : 0, to); + writeString(statePath, to); + writeCommitIntervals(intervals, to); processStderr(); readInt(from); } -IntVector RemoteStore::getStatePathsInterval(const PathSet & statePaths) +CommitIntervals RemoteStore::getStatePathsInterval(const Path & statePath) { writeInt(wopGetStatePathsInterval, to); - writeStringSet(statePaths, to); + writeString(statePath, to); processStderr(); - return readIntVector(from); + return readCommitIntervals(from); } bool RemoteStore::isStateComponent(const Path & path) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 77daae195..54ec72c41 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -76,9 +76,9 @@ public: void collectGarbage(GCAction action, const PathSet & pathsToDelete, bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed); - void setStatePathsInterval(const PathSet & statePath, const IntVector & intervals, bool allZero = false); + void setStatePathsInterval(const Path & statePath, const CommitIntervals & intervals); - IntVector getStatePathsInterval(const PathSet & statePaths); + CommitIntervals getStatePathsInterval(const Path & statePath); bool isStateComponent(const Path & path); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 877d99cda..7585088cc 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -183,10 +183,10 @@ public: bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed) = 0; /* TODO */ - virtual void setStatePathsInterval(const PathSet & statePath, const IntVector & intervals, bool allZero = false) = 0; + virtual void setStatePathsInterval(const Path & statePath, const CommitIntervals & intervals) = 0; /* TODO */ - virtual IntVector getStatePathsInterval(const PathSet & statePaths) = 0; + virtual CommitIntervals getStatePathsInterval(const Path & statePath) = 0; /* Checks whether a path is a component path that has a statePath. */ virtual bool isStateComponent(const Path & path) = 0; diff --git a/src/libstore/store-state.cc b/src/libstore/store-state.cc index 7cda932db..2a6c97b0e 100644 --- a/src/libstore/store-state.cc +++ b/src/libstore/store-state.cc @@ -21,32 +21,17 @@ namespace nix { -/* -void updatedStateDerivation(Path storePath) -{ - //We dont remove the old .svn folders - //update in database? - - printMsg(lvlTalkative, format("Resetting state drv settings")); -} -*/ +/* + * This gets called before a stateDir might be valid, so we get the data + * from the drv and not from the database + */ void createSubStateDirsTxn(const Transaction & txn, const DerivationStateOutputDirs & stateOutputDirs, const DerivationStateOutputs & stateOutputs) { Path statePath = stateOutputs.find("state")->second.statepath; string stateDir = statePath; PathSet intervalPaths; - - //check if we can create state dirs - //TODO - - /* - if( ! DirectoryExists( ....... ) ){ - } - else - printMsg(lvlTalkative, format("Statedir %1% already exists, so dont ........ ???? ") % ...); - */ for (DerivationStateOutputDirs::const_reverse_iterator i = stateOutputDirs.rbegin(); i != stateOutputDirs.rend(); ++i){ DerivationStateOutputDir d = i->second; @@ -68,12 +53,19 @@ void createSubStateDirsTxn(const Transaction & txn, const DerivationStateOutputD setChown(statePath, queryCallingUsername(), "nixbld", true); //Set all dirs in the statePath recursively to their owners printMsg(lvlTalkative, format("Set CHOWN '%1%'") % (statePath + "-" + queryCallingUsername())); - - //Initialize the counters for the statePaths that have an interval to 0 - IntVector empty; - setStatePathsIntervalTxn(txn, intervalPaths, empty, true); } +/* + * TODO + */ +void updatePaths() +{ + //set in db + + //update setStatePathsIntervalTxn(txn, intervalPaths, empty, true); +} + + /* * Input: store (or statePath?) * Returns all the drv's of the statePaths (in)directly referenced. @@ -143,7 +135,7 @@ void revertToRevisionTxn(const Transaction & txn, const Path & statePath, const if(revertPathOrFile.substr(revertPathOrFile.length() -1 , revertPathOrFile.length()) == "/"){ revertPathOrFile_e = revertPathOrFile.substr(0 , revertPathOrFile.length() -1) + "@" + unsignedInt2String(epoch) + "/"; - //TODO IF IS FILE: REMOVE THE FILE + //TODO IF IS FILE: REMOVE THE FILE todo MOVE THIS INTO RSYNCPATHS??? } else{ revertPathOrFile_e = revertPathOrFile + "@" + unsignedInt2String(epoch); @@ -175,65 +167,55 @@ void revertToRevisionTxn(const Transaction & txn, const Path & statePath, const -//TODO maybe add user ID ? Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath) { if(!isValidStatePathTxn(txn, statePath)) throw Error(format("path `%1%' is not a valid state path") % statePath); - //queryDeriversStatePath?? - Derivation drv = derivationFromPathTxn(txn, queryStatePathDrvTxn(txn, statePath)); - DerivationStateOutputs stateOutputs = drv.stateOutputs; - DerivationStateOutputDirs stateOutputDirs = drv.stateOutputDirs; - printMsg(lvlError, format("Snapshotting statePath: %1%") % statePath); - //Get all the inverals from the database at once - PathSet intervalPaths; - for (DerivationStateOutputDirs::const_reverse_iterator i = stateOutputDirs.rbegin(); i != stateOutputDirs.rend(); ++i){ - DerivationStateOutputDir d = i->second; - - string thisdir = d.path; - string fullstatedir = statePath + "/" + thisdir; - - if(d.type == "interval"){ - intervalPaths.insert(fullstatedir); - } - } - IntVector intervals = getStatePathsIntervalTxn(txn, intervalPaths); + //Get all paths from the db + StateInfos infos; + getVersionedStateEntriesTxn(txn, statePath, infos); + //Get all interval counters + CommitIntervals intervals = getStatePathsIntervalTxn(txn, statePath); + Snapshots revisions_list; - int intervalAt=0; - for (DerivationStateOutputDirs::const_iterator i = stateOutputDirs.begin(); i != stateOutputDirs.end(); ++i){ - DerivationStateOutputDir d = i->second; - string thisdir = d.path; + for (StateInfos::const_iterator i = infos.begin(); i != infos.end(); ++i){ - string fullstatedir = statePath + "/" + thisdir; - if(thisdir == "/") //exception for the root dir - fullstatedir = statePath + "/"; + string thisdir = (*i).path; + string type = (*i).type; + unsigned int interval = (*i).interval; - if(d.type == "none"){ + if(type == "none"){ continue; } - if(d.type == "interval"){ - //Get the interval-counter from the database - int interval_counter = intervals[intervalAt]; - int interval = d.getInterval(); - - //update the interval - intervals[intervalAt] = interval_counter + 1; - intervalAt++; - - if(interval_counter % interval != 0){ continue; } - } - else if(d.type == "full"){ } - else if(d.type == "manual"){ continue; } //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - else - throw Error(format("Type '%1%' is not handled in nix-state") % d.type); + if(type == "interval"){ + unsigned int interval_counter = intervals[thisdir]; - //We got here so we need to commit + //update the interval + intervals[thisdir] = (interval_counter + 1); + + //We continue if we dont have to commit now + if(interval_counter % interval != 0) + continue; + } + else if(type == "full"){ } + else if(type == "manual"){ continue; } //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + else + throw Error(format("Type '%1%' is not handled in nix-state") % type); + + /////////// We got here so we need to commit + + //We get all info from the orignal path, but we snapshot on the shared path + Path sharedWith = toNonSharedPathTxn(txn, statePath); + string fullstatedir = sharedWith + "/" + thisdir; + if(thisdir == "/") //exception for the root dir + fullstatedir = sharedWith + "/"; + unsigned int revision_number; if(pathExists(fullstatedir) || FileExist(fullstatedir)){ revision_number = take_snapshot(fullstatedir); @@ -245,11 +227,11 @@ Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath) revisions_list[fullstatedir] = revision_number; //printMsg(lvlError, format("FSD %1% RN %2%") % fullstatedir % revision_number); } - + + //Update the intervals again + setStatePathsIntervalTxn(txn, statePath, intervals); + return revisions_list; - - //Update the intervals again - //setStatePathsIntervalTxn(txn, intervalPaths, intervals); //TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!! uncomment } //TODO include this call in the validate function @@ -703,6 +685,7 @@ bool queryStateRevisions(Database & nixDB, const Transaction & txn, TableId revi splitDBRevKey(*i, getStatePath, getTimestamp); //query state versioined directorys/files + //TODO REMOVE /* vector sortedPaths; Derivation drv = derivationFromPathTxn(txn, queryStatePathDrvTxn(txn, getStatePath)); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 1662f3c2b..4a7225d95 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -105,6 +105,17 @@ void writeRevisionInfos(const RevisionInfos & ri, Sink & sink) } } +void writeCommitIntervals(const CommitIntervals ci, Sink & sink) +{ + writeInt(ci.size(), sink); + for (CommitIntervals::const_iterator i = ci.begin(); i != ci.end(); ++i){ + writeString((*i).first, sink); + writeBigUnsignedInt((*i).second, sink); + } +} + +//////////////// + void readPadding(unsigned int len, Source & source) { if (len % 8) { @@ -205,7 +216,7 @@ RevisionClosureTS readRevisionClosureTS(Source & source) RevisionClosureTS rc; while (count--){ string path = readString(source); - int ri = readBigUnsignedInt(source); + unsigned int ri = readBigUnsignedInt(source); rc[path] = ri; } return rc; @@ -225,4 +236,16 @@ RevisionInfos readRevisionInfos(Source & source) return ri; } +CommitIntervals readCommitIntervals(Source & source) +{ + unsigned int count = readInt(source); + CommitIntervals ci; + while (count--){ + string path = readString(source); + unsigned int ri = readBigUnsignedInt(source); + ci[path] = ri; + } + return ci; +} + } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index c6048e549..4f8bd65f7 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -104,6 +104,7 @@ void writeRevisionClosure(const RevisionClosure & rc, Sink & sink); void writeSnapshots(const Snapshots & ss, Sink & sink); void writeRevisionClosureTS(const RevisionClosureTS & rc, Sink & sink); void writeRevisionInfos(const RevisionInfos & ri, Sink & sink); +void writeCommitIntervals(const CommitIntervals ci, Sink & sink); void readPadding(unsigned int len, Source & source); unsigned int readInt(Source & source); @@ -116,6 +117,7 @@ RevisionClosure readRevisionClosure(Source & source); Snapshots readSnapshots(Source & source); RevisionClosureTS readRevisionClosureTS(Source & source); RevisionInfos readRevisionInfos(Source & source); +CommitIntervals readCommitIntervals(Source & source); } diff --git a/src/libutil/types.hh b/src/libutil/types.hh index f32ae2add..123fcb01e 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -65,6 +65,7 @@ typedef set PathSet; //state types typedef vector IntVector; //the Strings (list) of StateReferences and this list are connected by position //TODO typedef vector UnsignedIntVector; +typedef map CommitIntervals; struct RevisionInfo { string comment; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index ccdf8327a..c9fa8af90 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -663,7 +663,7 @@ static void installDerivations(Globals & globals, //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //TODO && (the users are equal || OR SHARED BETWEEN USERS) isnt this already user specific? - //TODO + //TODO if( newSharedState == "" && (oldStateIdentifier == newStateIdentifier diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index 57e4c9308..eaeeb29d3 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -374,9 +374,6 @@ static void opRunComponent(Strings opFlags, Strings opArgs) //Start transaction TODO - //Replace all shared paths in the set for their real paths - statePaths = store->toNonSharedPathSet(statePaths); - //Commit all statePaths RevisionClosure rivisionMapping; for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) //TODO first commit own state path? diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 8f8219eae..9c7b10e2a 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -473,22 +473,21 @@ static void performOp(Source & from, Sink & to, unsigned int op) } case wopSetStatePathsInterval: { - PathSet statePaths = readStringSet(from); - IntVector intervals = readIntVector(from); - bool allZero = readInt(from) == 1; + Path statePath = readStatePath(from); + CommitIntervals intervals = readCommitIntervals(from); startWork(); - store->setStatePathsInterval(statePaths, intervals, allZero); + store->setStatePathsInterval(statePath, intervals); stopWork(); writeInt(1, to); break; } case wopGetStatePathsInterval: { - PathSet statePaths = readStringSet(from); + Path statePath = readStatePath(from); startWork(); - IntVector iv = store->getStatePathsInterval(statePaths); + CommitIntervals ci = store->getStatePathsInterval(statePath); stopWork(); - writeIntVector(iv, to); + writeCommitIntervals(ci, to); break; }