From bdcce95a390f7648e956d1ab83f491310fbd2caf Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Mon, 27 Aug 2007 13:09:24 +0000 Subject: [PATCH] Added / Removed state functions to the Store API --- corepkgs/buildenv/builder.pl.in | 10 +- install_full.sh | 17 +- install_make.sh | 6 - src/libexpr/eval.hh | 2 +- src/libexpr/primops.cc | 11 +- src/libstore/build.cc | 6 +- src/libstore/gc.cc | 2 +- src/libstore/globals.cc | 34 +++- src/libstore/globals.hh | 11 +- src/libstore/local-store.cc | 69 ++++++-- src/libstore/local-store.hh | 27 ++- src/libstore/remote-store.cc | 47 +++++- src/libstore/remote-store.hh | 12 +- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 18 +- src/libstore/store-state.cc | 244 ++++++++++++++++++++++++++- src/libstore/store-state.hh | 10 +- src/libstore/worker-protocol.hh | 2 + src/libutil/util.cc | 17 +- src/libutil/util.hh | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-state/Makefile.am | 2 +- src/nix-state/nix-state.cc | 287 ++++---------------------------- src/nix-store/nix-store.cc | 4 +- src/nix-worker/nix-worker.cc | 163 +++++++++++++----- 25 files changed, 624 insertions(+), 383 deletions(-) diff --git a/corepkgs/buildenv/builder.pl.in b/corepkgs/buildenv/builder.pl.in index 917efc4a5..590d31ad6 100755 --- a/corepkgs/buildenv/builder.pl.in +++ b/corepkgs/buildenv/builder.pl.in @@ -121,10 +121,12 @@ sub createLinks { sysopen (DSTFILEHANDLE, $new_dstFile, O_RDWR|O_EXCL|O_CREAT, 0755); printf DSTFILEHANDLE "#! @shell@ \n"; - if($pkgRuntimeStateArgs eq "__NOARGS__") - { printf DSTFILEHANDLE "$nixBinDir/nix-state --run --identifier=$new_stateIdentifier $srcFile \"\$@\" \n"; } - else - { printf DSTFILEHANDLE "$nixBinDir/nix-state --run --identifier=$new_stateIdentifier $srcFile \"\$@\" $pkgRuntimeStateArgs \n"; } # TODO, maybe first R.T.A. then other args ? + if($pkgRuntimeStateArgs eq "__NOARGS__"){ + printf DSTFILEHANDLE "$nixBinDir/nix-state --run --identifier=$new_stateIdentifier $srcFile \"\$@\" \n"; + } + else{ + printf DSTFILEHANDLE "$nixBinDir/nix-state --run --identifier=$new_stateIdentifier $srcFile \"\$@\" $pkgRuntimeStateArgs \n"; + } # TODO, maybe first R.T.A. then other args ? close (DSTFILEHANDLE); } diff --git a/install_full.sh b/install_full.sh index 9cc626663..4d88ce283 100755 --- a/install_full.sh +++ b/install_full.sh @@ -1,7 +1,7 @@ #! /bin/sh -e -export nixstatepath=/nixstate/nix -export ACLOCAL_PATH=/root/.nix-profile/share/aclocal +export nixstatepath=/nixstate2/nix +export ACLOCAL_PATH=/home/wouterdb/.nix-profile/share/aclocal if [ "$1" = "full" ]; then nix-env-all-pkgs.sh -i gcc @@ -19,7 +19,8 @@ if [ "$1" = "full" ]; then nix-env-all-pkgs.sh -i docbook5-xsl nix-env-all-pkgs.sh -i bison nix-env-all-pkgs.sh -i gdb #optional for debugging - nix-env-all-pkgs.sh -i e3cfsprogs + nix-env-all-pkgs.sh -i gnupatch + nix-env-all-pkgs.sh -i gnumake fi if [ "$1" = "full" ] || [ "$1" = "auto" ]; then @@ -39,8 +40,8 @@ fi --with-bzip2=$HOME/.nix-profile \ --with-bdb=$HOME/.nix-profile \ --with-docbook-xsl=$HOME/.nix-profile \ - --with-docbook-rng=/root/.nix-profile/xml/rng/docbook \ - --with-docbook-xsl=/root/.nix-profile/xml/xsl/docbook \ + --with-docbook-rng=/home/wouterdb/.nix-profile/xml/rng/docbook \ + --with-docbook-xsl=/home/wouterdb/.nix-profile/xml/xsl/docbook \ --prefix=$nixstatepath \ --with-store-dir=/nix/store \ --with-store-state-dir=/nix/state \ @@ -61,9 +62,3 @@ fi echo "New state nix version by wouter ..." > doc/manual/NEWS.txt make -make install - -#for i in $nixstatepath/bin/*; do -# echo "pathing $i" -# patchelf --set-rpath ../lib/nix/:$(patchelf --print-rpath $i) $i -#done diff --git a/install_make.sh b/install_make.sh index 97ee176e9..fd6da82dc 100755 --- a/install_make.sh +++ b/install_make.sh @@ -1,11 +1,5 @@ #! /bin/sh -e make -make install -#export nixstatepath=/nixstate/nix -#for i in $nixstatepath/bin/*; do -# echo "pathing $i" -# patchelf --set-rpath ../lib/nix/:$(patchelf --print-rpath $i) $i -#done diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 5699d455c..7f4abe539 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -16,7 +16,7 @@ class Hash; typedef std::map DrvRoots; typedef std::map DrvHashes; -/* Cache for calls to addToStore(); maps source paths to the store +/* Cache for calls to addToStore(); maps source paths to the store //THIS OK ???? paths. */ typedef std::map SrcToStore; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 47d60de1a..d99d2b6fb 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -613,10 +613,15 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args) * We also add enableState to make it parse the drv to a state-drv * We also add runtimeStateArgs for the hash calc in hashDerivationModulo(...) to check if its needs to take the stateIdentiefier into account in the hash */ + + //TODO CHECK DRVS ARE CREATED BY THE USER AND NOT DAEMON ??????????????????????????? + //queryCallingUsername() + //THERE ARE 2 CALLS TO DerivationStateOutput below !!!!!!!!!!!!!!!!!!!!!! + if(enableState && !disableState){ if(runtimeStateArgs == ""){ string enableStateS = bool2string("true"); - drv.stateOutputs["state"] = DerivationStateOutput("", "", "", "", stateIdentifier, enableStateS, "", "", "", runtimeStateArgs, getCallingUserName(), "", false); + drv.stateOutputs["state"] = DerivationStateOutput("", "", "", "", stateIdentifier, enableStateS, "", "", "", runtimeStateArgs, queryCallingUsername(), "", false); } } @@ -652,7 +657,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args) string enableStateS = bool2string("true"); string createDirsBeforeInstallS = bool2string(createDirsBeforeInstall); drv.stateOutputs["state"] = DerivationStateOutput(stateOutPath, printHash(componentHash), outputHashAlgo, outputHash, stateIdentifier, enableStateS, - shareType, syncState, createDirsBeforeInstallS, runtimeStateArgs, getCallingUserName(), sharedState); + shareType, syncState, createDirsBeforeInstallS, runtimeStateArgs, queryCallingUsername(), sharedState); for(vector::iterator i = stateDirs.begin(); i != stateDirs.end(); ++i) drv.stateOutputDirs[(*i).path] = *(i); @@ -694,7 +699,7 @@ static Expr prim_derivationLazy(EvalState & state, const ATermVector & args) makeAttrRHS(makeSelect(drvStrict, toATerm("outPath")), makeNoPos())); attrs.set(toATerm("drvPath"), makeAttrRHS(makeSelect(drvStrict, toATerm("drvPath")), makeNoPos())); - attrs.set(toATerm("statePath"), //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! add a state check ! + attrs.set(toATerm("statePath"), makeAttrRHS(makeSelect(drvStrict, toATerm("statePath")), makeNoPos())); return makeAttrs(attrs); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d8c1e2d20..2b01ea129 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1079,7 +1079,7 @@ static string makeValidityRegistration(const PathSet & paths, for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) { s += *i + "\n"; - Path deriver = showDerivers ? queryDeriver(noTxn, *i) : ""; + Path deriver = showDerivers ? store->queryDeriver(*i) : ""; s += deriver + "\n"; PathSet references; @@ -1400,7 +1400,7 @@ void DerivationGoal::startBuilder() checkStatePath(drv); if(drv.stateOutputs.find("state")->second.getCreateDirsBeforeInstall()) - createStateDirs(drv.stateOutputDirs, drv.stateOutputs); + createStateDirsTxn(noTxn, drv.stateOutputDirs, drv.stateOutputs); } /* For convenience, set an environment pointing to the top build @@ -1640,7 +1640,7 @@ void DerivationGoal::computeClosure() //We create state dirs only when state is enabled and when the dirs need to be created after the installation if(drv.stateOutputs.size() != 0) if(!drv.stateOutputs.find("state")->second.getCreateDirsBeforeInstall()) - createStateDirs(drv.stateOutputDirs, drv.stateOutputs); + createStateDirsTxn(noTxn, drv.stateOutputDirs, drv.stateOutputs); /* for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ab4347e49..4e94b8f17 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -478,7 +478,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, /* Note that the deriver need not be valid (e.g., if we previously ran the collector with `gcKeepDerivations' turned off). */ - Path deriver = queryDeriver(noTxn, *i); + Path deriver = store->queryDeriver(*i); if (deriver != "" && store->isValidPath(deriver)) computeFSClosure(deriver, livePaths, true, false, -1); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WE (MAY) ALSO NEED TO KEEP STATE } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 980bd367f..4575d96b2 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -3,7 +3,7 @@ #include #include - +#include namespace nix { @@ -27,8 +27,9 @@ unsigned int maxBuildJobs = 1; bool readOnlyMode = false; string thisSystem = "unset"; unsigned int maxSilentTime = 0; - static bool settingsRead = false; +uid_t callingUID = 0; +bool debugWorker = true; static std::map settings; @@ -115,5 +116,32 @@ unsigned int queryIntSetting(const string & name, unsigned int def) return n; } - +uid_t queryCallingUID() +{ + /* A root user will not even bother calling the daemon, so there is no way to check + * If the uid is not yet set... + */ + return callingUID; +} + +void setCallingUID(uid_t uid, bool reset) +{ + if(callingUID != 0 && !reset) + throw Error(format("The UID of the caller aleady set! at this point")); + + callingUID = uid; +} + +string queryCallingUsername() +{ + uid_t uid = queryCallingUID(); + + passwd *pwd = getpwuid(uid); + char *pw_name = pwd->pw_name; + + return (string)pw_name; +} + + + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index fc3b1a78d..a85c2f63c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -80,7 +80,16 @@ bool queryBoolSetting(const string & name, bool def); unsigned int queryIntSetting(const string & name, unsigned int def); - +/* TODO PRIVATE: UID of the user that calls the nix-worker daemon */ +extern uid_t callingUID; + +/* get/set the UID of the user that calls the nix-worker daemon */ +uid_t queryCallingUID(); +void setCallingUID(uid_t uid, bool reset = false); +/* get the username based on the UID of the user that calls the nix-worker daemon */ +string queryCallingUsername(); + +extern bool debugWorker; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3d69bc952..e0532ce2f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -256,11 +256,11 @@ LocalStore::LocalStore(bool reserveSpace) throw Error(format("`%1%' is corrupt") % schemaFN); } - if (curSchema > nixSchemaVersion) + if (curSchema > nixSchemaVersion && curSchema != 4) //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MAJOR HACK, I SHOULD MERGE WITH THE TRUNK throw Error(format("current Nix store schema is version %1%, but I only support %2%") % curSchema % nixSchemaVersion); - if (curSchema < nixSchemaVersion) { + if (curSchema < nixSchemaVersion && curSchema != 4) { //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MAJOR HACK, I SHOULD MERGE WITH THE TRUNK if (curSchema <= 1) upgradeStore07(); if (curSchema == 2) @@ -809,7 +809,7 @@ bool LocalStore::isStateDrv(const Derivation & drv) return nix::isStateDrvTxn(noTxn, drv); } -Path queryDeriver(const Transaction & txn, const Path & storePath) +static Path queryDeriver(const Transaction & txn, const Path & storePath) { if (!isRealisablePath(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); @@ -827,6 +827,11 @@ Path queryDeriver(const Transaction & txn, const Path & storePath) return ""; } +Path LocalStore::queryDeriver(const Path & path) +{ + return nix::queryDeriver(noTxn, path); +} + //A '*' as argument stands for all identifiers or all users PathSet queryDerivers(const Transaction & txn, const Path & storePath, const string & identifier, const string & user) { @@ -859,6 +864,11 @@ PathSet queryDerivers(const Transaction & txn, const Path & storePath, const str return filtereddata; } +PathSet LocalStore::queryDerivers(const Path & storePath, const string & identifier, const string & user) +{ + return nix::queryDerivers(noTxn, storePath, identifier, user); +} + //Wrapper around converting the drvPath to the statePath /* PathSet queryDeriversStatePath(const Transaction & txn, const Path & storePath, const string & identifier, const string & user) @@ -1158,10 +1168,11 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed, registerValidPath(txn, dstPath, h, PathSet(), PathSet(), "", -1); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!! CHECK (probabyly ok?) txn.commit(); } + outputLock.setDeletion(true); } - + return dstPath; } @@ -1256,7 +1267,7 @@ void LocalStore::exportPath(const Path & path, bool sign, nix::queryXReferencesTxn(txn, path, references, true, -1); //TODO we can only now export the final revision //TODO also export the state references ??? writeStringSet(references, hashAndWriteSink); - Path deriver = queryDeriver(txn, path); + Path deriver = nix::queryDeriver(txn, path); writeString(deriver, hashAndWriteSink); if (sign) { @@ -1423,8 +1434,7 @@ void deleteFromStore(const Path & _path, unsigned long long & bytesFreed) throw PathInUse(format("cannot delete path `%1%' because it is in use by path `%2%'") % path % *i); invalidatePath(txn, path); - //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - //Also delete/invalidate stateReferrers????? + //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Also delete/invalidate stateReferrers????? } txn.commit(); @@ -1563,13 +1573,11 @@ void verifyStore(bool checkContents) txn.commit(); } -void setStatePathsInterval(const PathSet & statePaths, const vector & intervals, bool allZero) +void setStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePaths, const vector & intervals, bool allZero) { if(!allZero && statePaths.size() != intervals.size()){ throw Error("the number of statepaths and intervals must be equal"); } - - Transaction txn(nixDB); int n=0; for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) @@ -1583,13 +1591,13 @@ void setStatePathsInterval(const PathSet & statePaths, const vector & inter nixDB.setString(txn, dbStateCounters, *i, int2String(interval)); n++; } - - txn.commit(); } void LocalStore::setStatePathsInterval(const PathSet & statePaths, const vector & intervals, bool allZero) { - nix::setStatePathsInterval(statePaths, intervals, allZero); + Transaction txn(nixDB); + nix::setStatePathsIntervalTxn(txn, statePaths, intervals, allZero); + txn.commit(); } vector getStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePaths) @@ -1664,7 +1672,7 @@ void LocalStore::storePathRequisites(const Path & storeOrstatePath, const bool i nix::storePathRequisites(storeOrstatePath, includeOutputs, paths, withComponents, withState, revision); } -void queryAllValidPaths(const Transaction & txn, PathSet & allComponentPaths, PathSet & allStatePaths) +void queryAllValidPathsTxn(const Transaction & txn, PathSet & allComponentPaths, PathSet & allStatePaths) { Paths allComponentPaths2; Paths allStatePaths2; @@ -1711,9 +1719,24 @@ bool LocalStore::queryAvailableStateRevisions(const Path & statePath, RevisionIn return nix::queryAvailableStateRevisionsTxn(noTxn, statePath, revisions); } -void LocalStore::commitStatePath(const Path & statePath) +Snapshots LocalStore::commitStatePath(const Path & statePath) { - nix::commitStatePathTxn(noTxn, statePath); + Transaction txn(nixDB); + Snapshots ss = nix::commitStatePathTxn(txn, statePath); + txn.commit(); + return ss; +} + +void LocalStore::scanAndUpdateAllReferences(const Path & statePath, const bool recursive) +{ + Transaction txn(nixDB); + if(recursive) + nix::scanAndUpdateAllReferencesRecusivelyTxn(txn, statePath); + else{ + PathSet empty; + nix::scanAndUpdateAllReferencesTxn(txn, statePath, empty, empty); + } + txn.commit(); } void setSolidStateReferencesTxn(const Transaction & txn, const Path & statePath, const PathSet & paths) @@ -1767,6 +1790,11 @@ PathSet toNonSharedPathSetTxn(const Transaction & txn, const PathSet & statePath return real_statePaths; } +PathSet LocalStore::toNonSharedPathSet(const PathSet & statePaths) +{ + return toNonSharedPathSetTxn(noTxn, statePaths); +} + void setStateComponentReferencesTxn(const Transaction & txn, const Path & statePath, const Strings & references, int revision, int timestamp) { nixDB.setStateReferences(txn, dbStateComponentReferences, dbStateRevisions, statePath, references, revision, timestamp); @@ -1819,6 +1847,15 @@ PathSet getSharedWithPathSetRecTxn(const Transaction & txn, const Path & statePa return getSharedWithPathSetRecTxn_private(txn, statePath_ns, empty); } + +void LocalStore::revertToRevision(Path & componentPath, Path & derivationPath, Path & statePath, int revision_arg, bool recursive) +{ + Transaction txn(nixDB); + revertToRevisionTxn(txn, componentPath, derivationPath, statePath, revision_arg, recursive); + txn.commit(); +} + + /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */ static void upgradeStore07() { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d9e6877d2..193241f52 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -87,6 +87,8 @@ public: void collectGarbage(GCAction action, const PathSet & pathsToDelete, bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed); + + ///////////////////////////// void setStatePathsInterval(const PathSet & statePath, const vector & intervals, bool allZero = false); @@ -106,7 +108,17 @@ public: bool queryAvailableStateRevisions(const Path & statePath, RevisionInfos & revisions); - void commitStatePath(const Path & statePath); + Snapshots commitStatePath(const Path & statePath); + + Path queryDeriver(const Path & path); //should these be in here ???? + + PathSet queryDerivers(const Path & storePath, const string & identifier, const string & user); //should these be in here ???? + + void scanAndUpdateAllReferences(const Path & statePath, const bool recursive); + + PathSet toNonSharedPathSet(const PathSet & statePaths); + + void revertToRevision(Path & componentPath, Path & derivationPath, Path & statePath, int revision_arg, bool recursive); }; @@ -174,16 +186,9 @@ void setReferences(const Transaction & txn, const Path & store_or_statePath, void setDeriver(const Transaction & txn, const Path & path, const Path & deriver); -/* Query the deriver of a store path. Return the empty string if no - deriver has been set. */ -Path queryDeriver(const Transaction & txn, const Path & path); - /* Query the derivers of a state-store path. */ PathSet queryDerivers(const Transaction & txn, const Path & storePath, const string & identifier, const string & user); -/* Query the derivers of a state path. (are there more then 1 for a statepath?) */ -//PathSet queryDeriversStatePath(const Transaction & txn, const Path & storePath, const string & identifier, const string & user); - /* Delete a value from the nixStore directory. */ void deleteFromStore(const Path & path, unsigned long long & bytesFreed); @@ -217,7 +222,7 @@ bool isStateDrvPathTxn(const Transaction & txn, const Path & drvPath); bool isStateDrvTxn(const Transaction & txn, const Derivation & drv); //TODO can this ????? -void queryAllValidPaths(const Transaction & txn, PathSet & allComponentPaths, PathSet & allStatePaths); +void queryAllValidPathsTxn(const Transaction & txn, PathSet & allComponentPaths, PathSet & allStatePaths); bool isValidStatePathTxn(const Transaction & txn, const Path & path); void queryXReferencesTxn(const Transaction & txn, const Path & path, PathSet & references, const bool component_or_state, const int revision, int timestamp = -1); @@ -245,6 +250,10 @@ Path toNonSharedPathTxn(const Transaction & txn, const Path & statePath); PathSet getSharedWithPathSetRecTxn(const Transaction & txn, const Path & statePath); void ensurePathTxn(const Transaction & txn, const Path & path); +vector getStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePaths); + +bool queryStateRevisionsTxn(const Transaction & txn, const Path & statePath, RevisionClosure & revisions, RevisionClosureTS & timestamps, const int revision); +void setStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePath, const vector & intervals, bool allZero = false); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 48982cab6..48381eff4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -39,6 +39,7 @@ RemoteStore::RemoteStore() { string remoteMode = getEnv("NIX_REMOTE"); + debug(format("Client remoteMode: '%1%'") % remoteMode); if (remoteMode == "slave") /* Fork off a setuid worker to do the privileged work. */ forkSlave(); @@ -280,7 +281,9 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, writeString(hashAlgo, to); dumpPath(srcPath, to, filter); processStderr(); - Path path = readStorePath(from); + printMsg(lvlInfo, format("REMOTESTORE: ADD TO STORE REMOTE 1")); + Path path = readStorePath(from); + printMsg(lvlInfo, format("REMOTESTORE: ADD TO STORE REMOTE 2")); return path; } @@ -294,8 +297,7 @@ Path RemoteStore::addTextToStore(const string & suffix, const string & s, writeStringSet(references, to); processStderr(); - Path path = readStorePath(from); - return path; + return readStorePath(from); } @@ -318,7 +320,7 @@ Path RemoteStore::importPath(bool requireSignature, Source & source) processStderr(0, &source); Path path = readStorePath(from); - return path; + return readStorePath(from); } @@ -487,7 +489,42 @@ bool RemoteStore::queryAvailableStateRevisions(const Path & statePath, RevisionI } //TODO -void RemoteStore::commitStatePath(const Path & statePath) +Snapshots RemoteStore::commitStatePath(const Path & statePath) +{ + Snapshots ss; + return ss; +} + +void RemoteStore::scanAndUpdateAllReferences(const Path & statePath, const bool recursive) +{ + +} + +Path RemoteStore::queryDeriver(const Path & path) +{ + writeInt(wopQueryDeriver, to); + writeString(path, to); + processStderr(); + return readStorePath(from); +} + +PathSet RemoteStore::queryDerivers(const Path & storePath, const string & identifier, const string & user) +{ + writeInt(wopQueryDerivers, to); + writeString(storePath, to); + writeString(identifier, to); + writeString(user, to); + processStderr(); + return readStorePaths(from); //TODO is this ok ?? +} + +PathSet RemoteStore::toNonSharedPathSet(const PathSet & statePaths) +{ + PathSet p; + return p; +} + +void RemoteStore::revertToRevision(Path & componentPath, Path & derivationPath, Path & statePath, int revision_arg, bool recursive) { } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9a3f655cb..beb63a04c 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -94,7 +94,17 @@ public: bool queryAvailableStateRevisions(const Path & statePath, RevisionInfos & revisions); - void commitStatePath(const Path & statePath); + Snapshots commitStatePath(const Path & statePath); + + Path queryDeriver(const Path & path); + + PathSet queryDerivers(const Path & storePath, const string & identifier, const string & user); + + void scanAndUpdateAllReferences(const Path & statePath, const bool recursive); + + PathSet toNonSharedPathSet(const PathSet & statePaths); + + void revertToRevision(Path & componentPath, Path & derivationPath, Path & statePath, int revision_arg, bool recursive); private: AutoCloseFD fdSocket; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2f985648c..b75e832f2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -122,7 +122,7 @@ Path makeStatePath(const string & componentHash, const string & suffix, const st string suffix_stateIdentifier = stateIdentifier; suffix_stateIdentifier = "-" + suffix_stateIdentifier; - string username = getCallingUserName(); //Can and Should NOT be faked + string username = queryCallingUsername(); //Should NOT be fake-able /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ string s = ":sha256:" + componentHash + ":" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a7c238dbf..59f3d8054 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -59,7 +59,7 @@ public: /* Checks whether a path is valid. */ virtual bool isValidPath(const Path & path) = 0; - /* TODO */ + /* Checks whether a state-path is valid. */ virtual bool isValidStatePath(const Path & path) = 0; /* TODO */ @@ -207,7 +207,7 @@ public: /* TODO */ virtual vector getStatePathsInterval(const PathSet & statePaths) = 0; - /* TODO */ + /* Checks whether a path is a component path that has a statePath. */ virtual bool isStateComponent(const Path & path) = 0; /* TODO */ @@ -229,7 +229,19 @@ public: virtual bool queryAvailableStateRevisions(const Path & statePath, RevisionInfos & revisions) = 0; /* TODO */ - virtual void commitStatePath(const Path & statePath) = 0; + virtual Snapshots commitStatePath(const Path & statePath) = 0; + + /* Query the deriver of a store path. Return the empty string if + no deriver has been set. */ + virtual Path queryDeriver(const Path & path) = 0; + + virtual PathSet queryDerivers(const Path & storePath, const string & identifier, const string & user) = 0; + + virtual void scanAndUpdateAllReferences(const Path & statePath, const bool recursive) = 0; + + virtual PathSet toNonSharedPathSet(const PathSet & statePaths) = 0; + + virtual void revertToRevision(Path & componentPath, Path & derivationPath, Path & statePath, int revision_arg, bool recursive) = 0; }; diff --git a/src/libstore/store-state.cc b/src/libstore/store-state.cc index 5f775015b..43013727e 100644 --- a/src/libstore/store-state.cc +++ b/src/libstore/store-state.cc @@ -15,10 +15,11 @@ #include "misc.hh" #include "archive.hh" #include "snapshot.hh" +#include "references.hh" namespace nix { - +/* void updatedStateDerivation(Path storePath) { //We dont remove the old .svn folders @@ -26,8 +27,9 @@ void updatedStateDerivation(Path storePath) printMsg(lvlTalkative, format("Resetting state drv settings")); } +*/ -void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const DerivationStateOutputs & stateOutputs) +void createStateDirsTxn(const Transaction & txn, const DerivationStateOutputDirs & stateOutputDirs, const DerivationStateOutputs & stateOutputs) { Path statePath = stateOutputs.find("state")->second.statepath; string stateDir = statePath; @@ -64,9 +66,138 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De //Initialize the counters for the statePaths that have an interval to 0 vector empty; - store->setStatePathsInterval(intervalPaths, empty, true); + setStatePathsIntervalTxn(txn, intervalPaths, empty, true); } +/* + * Input: store (or statePath?) + * Returns all the drv's of the statePaths (in)directly referenced. + */ + //TODO TXN +PathSet getAllStateDerivationsRecursivelyTxn(const Transaction & txn, const Path & storePath, const int revision) +{ + //Get recursively all state paths + PathSet statePaths; + storePathRequisitesTxn(noTxn, storePath, false, statePaths, false, true, revision); + + //Find the matching drv with the statePath + PathSet derivations; + for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) + derivations.insert(queryStatePathDrvTxn(txn,*i)); + + return derivations; +} + + + +void revertToRevisionTxn(const Transaction & txn, Path & componentPath, Path & derivationPath, Path & statePath, int revision_arg, bool recursive) +{ + + PathSet statePaths; + if(recursive) + PathSet statePaths = getAllStateDerivationsRecursivelyTxn(txn, componentPath, revision_arg); //get dependecies (if neccecary | recusively) of all state components that need to be updated + else + statePaths.insert(derivationPath); //Insert direct state path + + //get a new timestamp for the references update + int newTimestamp = getTimeStamp(); + + //Get the revisions recursively to also roll them back + RevisionClosure getRivisions; + RevisionClosureTS getTimestamps; + queryStateRevisionsTxn(txn, statePath, getRivisions, getTimestamps, revision_arg); + + //Revert each statePath in the list + for (RevisionClosure::iterator i = getRivisions.begin(); i != getRivisions.end(); ++i){ + Path statePath = (*i).first; + Snapshots revisioned_paths = (*i).second; + int timestamp = getTimestamps[statePath]; + + //get its derivation-state-items + Derivation statePath_drv = derivationFromPathTxn(txn, queryStatePathDrvTxn(noTxn, statePath)); + DerivationStateOutputDirs stateOutputDirs = statePath_drv.stateOutputDirs; + + //TODO Sort snapshots??? eg first restore root, then the subdirs?? + + for (Snapshots::iterator j = revisioned_paths.begin(); j != revisioned_paths.end(); ++j){ + Path revertPathOrFile = (*j).first; + unsigned int epoch = (*j).second; + + //printMsg(lvlError, format("MAYBE '%1%'") % revertPathOrFile); + + //Look up the type from the drv with for the current snapshotted path + Path statePath_postfix = revertPathOrFile.substr(nixStoreState.length() + 1, revertPathOrFile.length() - nixStoreState.length()); + statePath_postfix = statePath_postfix.substr(statePath_postfix.find_first_of("/") + 1, statePath_postfix.length()); + if(statePath_postfix == "") + statePath_postfix = "/"; + string type = stateOutputDirs.at(statePath_postfix).type; + if(type == "none") + continue; + + //Now that were still here, we need to copy the state from the previous version back + Strings p_args; + p_args.push_back("-c"); //we use the shell to execute the cp command becuase the shell expands the '*' + string cpcommand = "cp"; + if(revertPathOrFile.substr(revertPathOrFile.length() -1 , revertPathOrFile.length()) == "/"){ //is dir + string revert_to_path = revertPathOrFile.substr(0, revertPathOrFile.length() -1) + "@" + unsignedInt2String(epoch); + cpcommand += " -R " + revert_to_path + "/*"; + + //clean all contents of the folder first (so were sure the path is clean) + if(pathExists(revertPathOrFile)) + deletePath(revertPathOrFile); + + if(epoch == 0) //Path was deleted so were done + continue; + else //If path was not deleted in the previous version, we need to make sure it exists or cp will fail + ensureDirExists(revertPathOrFile); + + //If the the dir has not contents then a cp ..../* will error since * cannot be expanded. So in this case were done and dont have to revert. + Strings p2_args; + p2_args.push_back("-A"); + p2_args.push_back(revert_to_path + "/"); + string output = runProgram("ls", true, p2_args); + if(output == "") + continue; + } + else{ //is file + cpcommand += " " + (revertPathOrFile + "@" + unsignedInt2String(epoch)); + + if(epoch == 0){ + //delete file + if(FileExist(revertPathOrFile)) + deletePath(revertPathOrFile); //we only delete if the cp doesnt overwrite it below + continue; + } + } + + //Revert + printMsg(lvlError, format("Reverting '%1%@%2%'") % revertPathOrFile % unsignedInt2String(epoch)); + printMsg(lvlError, format("Command: '%1%'") % cpcommand); + cpcommand += " " + revertPathOrFile; + p_args.push_back(cpcommand); + runProgram_AndPrintOutput("sh", true, p_args, "sh-cp"); + } + + + //*** Now also revert state references to the specific revision (the revision is already converted to a timestamp here) + + //Query the references of the old revision (already converted to a timestamp) + PathSet state_references; + queryXReferencesTxn(txn, statePath, state_references, true, -1, timestamp); + PathSet state_stateReferences; + queryXReferencesTxn(txn, statePath, state_stateReferences, false, -1, timestamp); + + //Now set these old references as the new references at the new (just created) Timestamp + setStateComponentReferencesTxn(txn, statePath, Strings(state_references.begin(), state_references.end()), -1, newTimestamp); + setStateStateReferencesTxn(txn, statePath, Strings(state_stateReferences.begin(), state_stateReferences.end()), -1, newTimestamp); + + printMsg(lvlError, format("Reverted state of '%1%' to revision '%2%'") % statePath % revision_arg); + } +} + + + +//TODO maybe add user ID ? Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath) { if(!isValidStatePathTxn(txn, statePath)) @@ -91,7 +222,7 @@ Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath) intervalPaths.insert(fullstatedir); } } - vector intervals = store->getStatePathsInterval(intervalPaths); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! txn + vector intervals = getStatePathsIntervalTxn(txn, intervalPaths); Snapshots revisions_list; @@ -142,4 +273,109 @@ Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath) //setStatePathsIntervalTxn(txn, intervalPaths, intervals); //TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!! uncomment } +//TODO include this call in the validate function +//TODO ONLY CALL THIS FUNCTION ON A NON-SHARED STATE PATH!!!!!!!!!!! +void scanAndUpdateAllReferencesTxn(const Transaction & txn, const Path & statePath + , PathSet & newFoundComponentReferences, PathSet & newFoundStateReferences) //only for recursion +{ + //Check if is a state Path + if(! isValidStatePathTxn(txn, statePath)) + throw Error(format("This path '%1%' is not a state path") % statePath); + + //printMsg(lvlError, format("scanAndUpdateAllReferencesTxn: '%1%' - %2%") % statePath % revision); + + //TODO check if path is not a shared path ! + //TODO + + //get all possible state and component references + PathSet allComponentPaths; + PathSet allStatePaths; + queryAllValidPathsTxn(txn, allComponentPaths, allStatePaths); + + //Remove derivation paths + PathSet allComponentPaths2; //without derivations + for (PathSet::iterator i = allComponentPaths.begin(); i != allComponentPaths.end(); ++i){ + string path = *i; + if(path.substr(path.length() - 4,path.length()) != drvExtension) //TODO HACK: we should have a typed table or a seperate table .... drvExtension == ".drv" + allComponentPaths2.insert(path); + } + + //TODO maybe only scan in the changeset (patch) for new references? (this will be difficult and depending on the underlying versioning system) + + //Scan in for (new) component and state references + PathSet state_references = scanForReferences(statePath, allComponentPaths2); + PathSet state_stateReferences = scanForStateReferences(statePath, allStatePaths); + + //Retrieve old references + PathSet old_references; + PathSet old_state_references; + queryXReferencesTxn(txn, statePath, old_references, true, -1); //get the latsest references + queryXReferencesTxn(txn, statePath, old_state_references, false, -1); + + //Check for added and removed paths + PathSet diff_references_removed; + PathSet diff_references_added; + pathSets_difference(state_references, old_references, diff_references_removed, diff_references_added); + PathSet diff_state_references_removed; + PathSet diff_state_references_added; + pathSets_difference(state_stateReferences, old_state_references, diff_state_references_removed, diff_state_references_added); + + //Set PathSet's for the caller of this function + newFoundComponentReferences = diff_references_added; + newFoundStateReferences = diff_state_references_added; + + //Print error, but we could also throw an error. + if(diff_references_added.size() != 0) + for (PathSet::iterator i = diff_references_added.begin(); i != diff_references_added.end(); ++i) + printMsg(lvlError, format("Added component reference found!: '%1%' in state path '%2%'") % (*i) % statePath); + if(diff_references_removed.size() != 0) + for (PathSet::iterator i = diff_references_removed.begin(); i != diff_references_removed.end(); ++i) + printMsg(lvlError, format("Removed component reference found!: '%1%' in state path '%2%'") % (*i) % statePath); + if(diff_state_references_added.size() != 0) + for (PathSet::iterator i = diff_state_references_added.begin(); i != diff_state_references_added.end(); ++i) + printMsg(lvlError, format("Added state reference found!: '%1%' in state path '%2%'") % (*i) % statePath); + if(diff_state_references_removed.size() != 0) + for (PathSet::iterator i = diff_state_references_removed.begin(); i != diff_state_references_removed.end(); ++i) + printMsg(lvlError, format("Removed state reference found!: '%1%' in state path '%2%'") % (*i) % statePath); + + //We always set the referernces so we know they were scanned (maybe the same) at a certain time + printMsg(lvlError, format("Updating new references for statepath: '%1%'") % statePath); + Path drvPath = queryStatePathDrvTxn(txn, statePath); + registerValidPath(txn, + statePath, + Hash(), //emtpy hash + state_references, + state_stateReferences, + drvPath, + -1); //Set at a new timestamp +} + +void scanAndUpdateAllReferencesRecusivelyTxn(const Transaction & txn, const Path & statePath) +{ + if(! isValidStatePathTxn(txn, statePath)) + throw Error(format("This path '%1%' is not a state path") % statePath); + + //get all state current state references recursively + PathSet statePaths; + storePathRequisitesTxn(txn, statePath, false, statePaths, false, true, -1); //Get all current state dependencies + + //Add own statePath (may already be in there, but its a set, so no doubles) + statePaths.insert(statePath); + + //We dont need to sort since the db does that + //call scanForAllReferences again on all statePaths + for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i){ + + //Scan, update, call recursively + PathSet newFoundComponentReferences; + PathSet newFoundStateReferences; + scanAndUpdateAllReferencesTxn(txn, *i, newFoundComponentReferences, newFoundStateReferences); + + //Call the function recursively again on all newly found references //TODO test if this works + PathSet allNewReferences = pathSets_union(newFoundComponentReferences, newFoundStateReferences); + for (PathSet::iterator j = allNewReferences.begin(); j != allNewReferences.end(); ++j) + scanAndUpdateAllReferencesRecusivelyTxn(txn, *j); + } +} + } diff --git a/src/libstore/store-state.hh b/src/libstore/store-state.hh index aacaf4eca..dbde130d3 100644 --- a/src/libstore/store-state.hh +++ b/src/libstore/store-state.hh @@ -8,7 +8,7 @@ namespace nix { /* Create a state directory. */ -void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const DerivationStateOutputs & stateOutputs); +void createStateDirsTxn(const Transaction & txn, const DerivationStateOutputDirs & stateOutputDirs, const DerivationStateOutputs & stateOutputs); /* TODO */ Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath); @@ -19,6 +19,14 @@ Snapshots commitStatePathTxn(const Transaction & txn, const Path & statePath); /* TODO */ //int readRevisionNumber(Path statePath); + +void scanAndUpdateAllReferencesTxn(const Transaction & txn, const Path & statePath + , PathSet & newFoundComponentReferences, PathSet & newFoundStateReferences); + +void scanAndUpdateAllReferencesRecusivelyTxn(const Transaction & txn, const Path & statePath); + +void revertToRevisionTxn(const Transaction & txn, Path & componentPath, Path & derivationPath, Path & statePath, int revision_arg, bool recursive); + } #endif /* !__STORESTATE_H */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 57f39ac36..b6dfff486 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -33,6 +33,8 @@ typedef enum { wopCollectGarbage, wopExportPath, wopImportPath, + wopQueryDeriver, + wopQueryDerivers, } WorkerOp; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 174f739d0..b12d4298e 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -370,8 +370,9 @@ void writeStringToFile(const Path & path, const string & s) LogType logType = ltPretty; -Verbosity verbosity = lvlInfo; +//Verbosity verbosity = lvlInfo; //Verbosity verbosity = lvlDebug; +Verbosity verbosity = lvlVomit; static int nestingLevel = 0; @@ -1176,20 +1177,9 @@ bool IsDirectory(const string FileName) return ((my_stat.st_mode & S_IFDIR) != 0); } - +/* string getCallingUserName() { - //TODO Make this work on WINDOWS: Untested! - /* - #include - char acUserName[100]; - DWORD nUserName = sizeof(acUserName); - if (GetUserName(acUserName, &nUserName)) - cout << "User name is " << acUserName << "." << endl; - else - cerr << "Failed to lookup user name, error code " << GetLastError() << "." << endl; - */ - //Linux Strings empty; string username = runProgram("whoami", true, empty); //the username of the user that is trying to build the component @@ -1200,6 +1190,7 @@ string getCallingUserName() return username; } +*/ //merges two PathSets into one, removing doubles (union) PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 73f6c6f89..0ad98e30f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -310,7 +310,7 @@ void runProgram_AndPrintOutput(Path program, bool searchPath, const Strings & ar int getTimeStamp(); -string getCallingUserName(); +//string getCallingUserName(); /* TODO */ PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index f9f97d674..fdbd1d65b 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -563,7 +563,7 @@ static void installDerivations(Globals & globals, string oldStatePath; //query old state path - PathSet derivers = queryDerivers(noTxn, i->queryOutPath(globals.state), newStateIdentifier, getCallingUserName()); + PathSet derivers = store->queryDerivers(i->queryOutPath(globals.state), newStateIdentifier, int2String(geteuid())); //TODO Check if username if ok if(derivers.size() != 1) throw Error(format("Internal Error: There is not exactly one deriver with state_identifier '%1%' for path '%2%'") % newStateIdentifier % i->queryOutPath(globals.state)); diff --git a/src/nix-state/Makefile.am b/src/nix-state/Makefile.am index 9be2b1caa..11d36a924 100644 --- a/src/nix-state/Makefile.am +++ b/src/nix-state/Makefile.am @@ -11,4 +11,4 @@ nix-state.o: help.txt.hh AM_CXXFLAGS = \ -I$(srcdir)/.. ${bdb_include} $(aterm_include) -I$(srcdir)/../libutil \ - -I$(srcdir)/../libstore -I$(srcdir)/../libmain + -I$(srcdir)/../libstore -I$(srcdir)/../libmain -I$(srcdir)/../libext3cow diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index 8996eba96..5eaf13335 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -14,6 +14,7 @@ #include "references.hh" #include "store-state.hh" #include "config.h" +#include "snapshot.hh" using namespace nix; using std::cin; @@ -67,9 +68,9 @@ Derivation getDerivation(const string & fullPath, const Strings & program_args, //printMsg(lvlError, format("'%1%' - '%2%' - '%3%' - '%4%' - '%5%'") % componentPath % state_identifier % binary % username % program_args); if(isStateComponent) - derivers = queryDerivers(noTxn, componentPath, state_identifier, username); + derivers = store->queryDerivers(componentPath, state_identifier, username); else - derivers.insert(queryDeriver(noTxn, componentPath)); + derivers.insert(store->queryDeriver(componentPath)); if(getDerivers == true) return Derivation(); @@ -162,24 +163,23 @@ static void opShowStatePath(Strings opFlags, Strings opArgs) printMsg(lvlError, format("%1%") % statePath); } -/* - * Input: store (or statePath?) - * Returns all the drv's of the statePaths (in)directly referenced. - */ -PathSet getAllStateDerivationsRecursively(const Path & storePath, const int revision) + +static void revertToRevision(Strings opFlags, Strings opArgs) { - //Get recursively all state paths - PathSet statePaths; - store->storePathRequisites(storePath, false, statePaths, false, true, revision); - - //Find the matching drv with the statePath - PathSet derivations; - for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) - derivations.insert(store->queryStatePathDrv(*i)); - - return derivations; + Path componentPath; + Path statePath; + string binary; + string derivationPath; + bool isStateComponent; + Strings program_args; + Derivation drv = getDerivation_andCheckArgs(opFlags, opArgs, componentPath, statePath, binary, derivationPath, isStateComponent, program_args); + + bool recursive = revert_recursively; + + store->revertToRevision(componentPath, derivationPath, statePath, revision_arg, recursive); } + static void queryAvailableStateRevisions(Strings opFlags, Strings opArgs) { Path statePath; @@ -196,7 +196,9 @@ static void queryAvailableStateRevisions(Strings opFlags, Strings opArgs) } //Unshare if neccacary - Path nonSharedStatePath = toNonSharedPathTxn(noTxn, statePath); + PathSet nonSharedPaths; + nonSharedPaths.insert(statePath); + Path nonSharedStatePath = *((store->toNonSharedPathSet(nonSharedPaths)).begin()); //TODO CHECK IF THIS WORKS !!!!!!!! if(nonSharedStatePath != statePath){ printMsg(lvlError, format("The statePath is shared with this path %1%") % nonSharedStatePath); statePath = nonSharedStatePath; @@ -204,8 +206,7 @@ static void queryAvailableStateRevisions(Strings opFlags, Strings opArgs) RevisionInfos revisions; bool notEmpty = store->queryAvailableStateRevisions(statePath, revisions); - - + if(!notEmpty){ printMsg(lvlError, format("No revisions yet for: %1%") % statePath); return; @@ -240,227 +241,6 @@ static void queryAvailableStateRevisions(Strings opFlags, Strings opArgs) } } -static void revertToRevision(Strings opFlags, Strings opArgs) -{ - Path componentPath; - Path statePath; - string binary; - string derivationPath; - bool isStateComponent; - Strings program_args; - Derivation drv = getDerivation_andCheckArgs(opFlags, opArgs, componentPath, statePath, binary, derivationPath, isStateComponent, program_args); - - bool recursive = revert_recursively; - - //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! add TXN here ??????????? - - PathSet statePaths; - if(recursive) - PathSet statePaths = getAllStateDerivationsRecursively(componentPath, revision_arg); //get dependecies (if neccecary | recusively) of all state components that need to be updated - else - statePaths.insert(derivationPath); //Insert direct state path - - //get a new timestamp for the references update - int newTimestamp = getTimeStamp(); - - //Get the revisions recursively to also roll them back - RevisionClosure getRivisions; - RevisionClosureTS getTimestamps; - bool b = store->queryStateRevisions(statePath, getRivisions, getTimestamps, revision_arg); - - //Revert each statePath in the list - for (RevisionClosure::iterator i = getRivisions.begin(); i != getRivisions.end(); ++i){ - Path statePath = (*i).first; - Snapshots revisioned_paths = (*i).second; - int timestamp = getTimestamps[statePath]; - - //get its derivation-state-items - Derivation statePath_drv = derivationFromPathTxn(noTxn, queryStatePathDrvTxn(noTxn, statePath)); - DerivationStateOutputDirs stateOutputDirs = statePath_drv.stateOutputDirs; - - //TODO Sort snapshots??? eg first restore root, then the subdirs?? - - for (Snapshots::iterator j = revisioned_paths.begin(); j != revisioned_paths.end(); ++j){ - Path revertPathOrFile = (*j).first; - unsigned int epoch = (*j).second; - - //printMsg(lvlError, format("MAYBE '%1%'") % revertPathOrFile); - - //Look up the type from the drv with for the current snapshotted path - Path statePath_postfix = revertPathOrFile.substr(nixStoreState.length() + 1, revertPathOrFile.length() - nixStoreState.length()); - statePath_postfix = statePath_postfix.substr(statePath_postfix.find_first_of("/") + 1, statePath_postfix.length()); - if(statePath_postfix == "") - statePath_postfix = "/"; - string type = stateOutputDirs.at(statePath_postfix).type; - if(type == "none") - continue; - - //Now that were still here, we need to copy the state from the previous version back - Strings p_args; - p_args.push_back("-c"); //we use the shell to execute the cp command becuase the shell expands the '*' - string cpcommand = "cp"; - if(revertPathOrFile.substr(revertPathOrFile.length() -1 , revertPathOrFile.length()) == "/"){ //is dir - string revert_to_path = revertPathOrFile.substr(0, revertPathOrFile.length() -1) + "@" + unsignedInt2String(epoch); - cpcommand += " -R " + revert_to_path + "/*"; - - //clean all contents of the folder first (so were sure the path is clean) - if(pathExists(revertPathOrFile)) - deletePath(revertPathOrFile); - - if(epoch == 0) //Path was deleted so were done - continue; - else //If path was not deleted in the previous version, we need to make sure it exists or cp will fail - ensureDirExists(revertPathOrFile); - - //If the the dir has not contents then a cp ..../* will error since * cannot be expanded. So in this case were done and dont have to revert. - Strings p2_args; - p2_args.push_back("-A"); - p2_args.push_back(revert_to_path + "/"); - string output = runProgram("ls", true, p2_args); - if(output == "") - continue; - } - else{ //is file - cpcommand += " " + (revertPathOrFile + "@" + unsignedInt2String(epoch)); - - if(epoch == 0){ - //delete file - if(FileExist(revertPathOrFile)) - deletePath(revertPathOrFile); //we only delete if the cp doesnt overwrite it below - continue; - } - } - - //Revert - printMsg(lvlError, format("Reverting '%1%@%2%'") % revertPathOrFile % unsignedInt2String(epoch)); - printMsg(lvlError, format("Command: '%1%'") % cpcommand); - cpcommand += " " + revertPathOrFile; - p_args.push_back(cpcommand); - runProgram_AndPrintOutput("sh", true, p_args, "sh-cp"); - } - - - //*** Now also revert state references to the specific revision (the revision is already converted to a timestamp here) - - //Query the references of the old revision (already converted to a timestamp) - PathSet state_references; - queryXReferencesTxn(noTxn, statePath, state_references, true, -1, timestamp); - PathSet state_stateReferences; - queryXReferencesTxn(noTxn, statePath, state_stateReferences, false, -1, timestamp); - - //Now set these old references as the new references at the new (just created) Timestamp - setStateComponentReferencesTxn(noTxn, statePath, Strings(state_references.begin(), state_references.end()), -1, newTimestamp); - setStateStateReferencesTxn(noTxn, statePath, Strings(state_stateReferences.begin(), state_stateReferences.end()), -1, newTimestamp); - - printMsg(lvlError, format("Reverted state of '%1%' to revision '%2%'") % statePath % revision_arg); - } -} - -//TODO include this call in the validate function -//TODO ONLY CALL THIS FUNCTION ON A NON-SHARED STATE PATH!!!!!!!!!!! -void scanAndUpdateAllReferencesTxn(const Transaction & txn, const Path & statePath - , PathSet & newFoundComponentReferences, PathSet & newFoundStateReferences) //only for recursion -{ - //Check if is a state Path - if(! isValidStatePathTxn(txn, statePath)) - throw Error(format("This path '%1%' is not a state path") % statePath); - - //printMsg(lvlError, format("scanAndUpdateAllReferencesTxn: '%1%' - %2%") % statePath % revision); - - //TODO check if path is not a shared path ! - //TODO - - //get all possible state and component references - PathSet allComponentPaths; - PathSet allStatePaths; - queryAllValidPaths(txn, allComponentPaths, allStatePaths); - - //Remove derivation paths - PathSet allComponentPaths2; //without derivations - for (PathSet::iterator i = allComponentPaths.begin(); i != allComponentPaths.end(); ++i){ - string path = *i; - if(path.substr(path.length() - 4,path.length()) != drvExtension) //TODO HACK: we should have a typed table or a seperate table .... drvExtension == ".drv" - allComponentPaths2.insert(path); - } - - //TODO maybe only scan in the changeset (patch) for new references? (this will be difficult and depending on the underlying versioning system) - - //Scan in for (new) component and state references - PathSet state_references = scanForReferences(statePath, allComponentPaths2); - PathSet state_stateReferences = scanForStateReferences(statePath, allStatePaths); - - //Retrieve old references - PathSet old_references; - PathSet old_state_references; - queryXReferencesTxn(txn, statePath, old_references, true, -1); //get the latsest references - queryXReferencesTxn(txn, statePath, old_state_references, false, -1); - - //Check for added and removed paths - PathSet diff_references_removed; - PathSet diff_references_added; - pathSets_difference(state_references, old_references, diff_references_removed, diff_references_added); - PathSet diff_state_references_removed; - PathSet diff_state_references_added; - pathSets_difference(state_stateReferences, old_state_references, diff_state_references_removed, diff_state_references_added); - - //Set PathSet's for the caller of this function - newFoundComponentReferences = diff_references_added; - newFoundStateReferences = diff_state_references_added; - - //Print error, but we could also throw an error. - if(diff_references_added.size() != 0) - for (PathSet::iterator i = diff_references_added.begin(); i != diff_references_added.end(); ++i) - printMsg(lvlError, format("Added component reference found!: '%1%' in state path '%2%'") % (*i) % statePath); - if(diff_references_removed.size() != 0) - for (PathSet::iterator i = diff_references_removed.begin(); i != diff_references_removed.end(); ++i) - printMsg(lvlError, format("Removed component reference found!: '%1%' in state path '%2%'") % (*i) % statePath); - if(diff_state_references_added.size() != 0) - for (PathSet::iterator i = diff_state_references_added.begin(); i != diff_state_references_added.end(); ++i) - printMsg(lvlError, format("Added state reference found!: '%1%' in state path '%2%'") % (*i) % statePath); - if(diff_state_references_removed.size() != 0) - for (PathSet::iterator i = diff_state_references_removed.begin(); i != diff_state_references_removed.end(); ++i) - printMsg(lvlError, format("Removed state reference found!: '%1%' in state path '%2%'") % (*i) % statePath); - - //We always set the referernces so we know they were scanned (maybe the same) at a certain time - printMsg(lvlError, format("Updating new references for statepath: '%1%'") % statePath); - Path drvPath = queryStatePathDrvTxn(txn, statePath); - registerValidPath(txn, - statePath, - Hash(), //emtpy hash - state_references, - state_stateReferences, - drvPath, - -1); //Set at a new timestamp -} - -void scanAndUpdateAllReferencesRecusivelyTxn(const Transaction & txn, const Path & statePath) -{ - if(! isValidStatePathTxn(txn, statePath)) - throw Error(format("This path '%1%' is not a state path") % statePath); - - //get all state current state references recursively - PathSet statePaths; - store->storePathRequisites(statePath, false, statePaths, false, true, -1); //Get all current state dependencies - - //Add own statePath (may already be in there, but its a set, so no doubles) - statePaths.insert(statePath); - - //We dont need to sort since the db does that - //call scanForAllReferences again on all statePaths - for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i){ - - //Scan, update, call recursively - PathSet newFoundComponentReferences; - PathSet newFoundStateReferences; - scanAndUpdateAllReferencesTxn(txn, *i, newFoundComponentReferences, newFoundStateReferences); - - //Call the function recursively again on all newly found references //TODO test if this works - PathSet allNewReferences = pathSets_union(newFoundComponentReferences, newFoundStateReferences); - for (PathSet::iterator j = allNewReferences.begin(); j != allNewReferences.end(); ++j) - scanAndUpdateAllReferencesRecusivelyTxn(txn, *j); - } -} - static void opRunComponent(Strings opFlags, Strings opArgs) @@ -482,9 +262,7 @@ static void opRunComponent(Strings opFlags, Strings opArgs) //add locks ... ? //svn lock ... ? - //TODO - Transaction txn; - //createStoreTransaction(txn); + //******************* Run **************************** @@ -506,10 +284,16 @@ static void opRunComponent(Strings opFlags, Strings opArgs) printMsg(lvlError, format("Command: '%1%'") % (root_componentPath + root_binary + root_args)); executeShellCommand(root_componentPath + root_binary + root_args); } + + ////////////////////////////// + + //TODO + Transaction txn; + //createStoreTransaction(txn); //******************* Scan for new references if neccecary if(scanforReferences) - scanAndUpdateAllReferencesRecusivelyTxn(txn, root_statePath); + store->scanAndUpdateAllReferences(root_statePath, true); //TODO make recursive a paramter? //get all current (maybe updated by the scan) dependecies (if neccecary | recusively) of all state components that need to be updated PathSet statePaths; @@ -519,17 +303,17 @@ static void opRunComponent(Strings opFlags, Strings opArgs) //Start transaction TODO //Replace all shared paths in the set for their real paths - statePaths = toNonSharedPathSetTxn(noTxn, statePaths); + statePaths = store->toNonSharedPathSet(statePaths); //******************* With everything in place, we call the commit script on all statePaths (in)directly referenced ********************** //Commit all statePaths RevisionClosure rivisionMapping; for (PathSet::iterator i = statePaths.begin(); i != statePaths.end(); ++i) //TODO first commit own state path? - rivisionMapping[*i] = commitStatePathTxn(txn, *i); + rivisionMapping[*i] = store->commitStatePath(*i); //Save new revisions - setStateRevisionsTxn(txn, rivisionMapping, root_statePath, comment); + store->setStateRevisions(rivisionMapping, root_statePath, comment); //TODO how about the txn? //Commit transaction //txn.commit(); @@ -576,7 +360,7 @@ void run(Strings args) store = openStore(); printMsg(lvlError, format("1: %1%") % bool2string( store->isStateComponent("/nix/store/7xkw5fkz5yw7dpx0pc6l12bh9a56135c-hellostateworld-1.0") ) ); printMsg(lvlError, format("2: %1%") % bool2string( store->isStateComponent("/nix/store/05441jm8xmsidqm43ivk0micckf0mr2m-nvidiaDrivers") ) ); - printMsg(lvlError, format("3: %1%") % bool2string( store->isStateDrvPath("/nix/store/2hpx60ibdfv2pslg4rjvp177frijamvi-hellostateworld-1.0.drv") ) ); + printMsg(lvlError, format("3: %1%") % bool2string( isState Drv Path("/nix/store/2hpx60ibdfv2pslg4rjvp177frijamvi-hellostateworld-1.0.drv") ) ); store = openStore(); Path p = store->queryStatePathDrv("/nix/state/6g6kfgimz8szznlshf13s29fn01zp99d-hellohardcodedstateworld-1.0-test2"); @@ -684,7 +468,7 @@ void run(Strings args) // /nix/state/6l93ff3bn1mk61jbdd34diafmb4aq7c6-hellohardcodedstateworld-1.0- // /nix/state/x8k4xiv8m4zmx26gmb0pyymmd6671fyy-hellohardcodedstateworld-1.0- - PathSet p = getSharedWithPathSetRecTxn(noTxn, "/nix/state/6l93ff3bn1mk61jbdd34diafmb4aq7c6-hellohardcodedstateworld-1.0-"); + PathSet p = store->getSharedWithPathSetRec("/nix/state/6l93ff3bn1mk61jbdd34diafmb4aq7c6-hellohardcodedstateworld-1.0-"); for (PathSet::iterator j = p.begin(); j != p.end(); ++j) printMsg(lvlError, format("P: '%1%'") % *j ); return; @@ -767,8 +551,9 @@ void run(Strings args) throw UsageError("only one operation may be specified"); } + //If no username given: take the username of the caller if(username == "") - username = getCallingUserName(); + username = int2String(geteuid()); if (!op) throw UsageError("no operation specified"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f7faa5c1d..4f79c5b8a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -59,7 +59,7 @@ static Path fixStoreOrStatePath(Path path) static Path useDeriver(Path path) { if (!isDerivation(path)) { - path = queryDeriver(noTxn, path); + path = store->queryDeriver(path); if (path == "") throw Error(format("deriver of path `%1%' is not known") % path); } @@ -331,7 +331,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) { - Path deriver = queryDeriver(noTxn, fixPath(*i)); + Path deriver = store->queryDeriver(fixPath(*i)); cout << format("%1%\n") % (deriver == "" ? "unknown-deriver" : deriver); } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 03c523d04..b237bb22a 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -43,7 +43,7 @@ static void tunnelStderr(const unsigned char * buf, size_t count) process handling the connection. Otherwise we could screw up the protocol. It's up to the parent to redirect stderr and send it to the client somehow (e.g., as in build.cc). */ - if (canSendStderr && myPid == getpid()) { + if (canSendStderr && myPid == getpid() && (!debugWorker)) { //TODO debugWorker commit try { writeInt(STDERR_NEXT, to); writeString(string((char *) buf, count), to); @@ -53,8 +53,10 @@ static void tunnelStderr(const unsigned char * buf, size_t count) canSendStderr = false; throw; } - } else + } else{ + //printMsg(lvlInfo, format("nix-worker: debug mode")); writeFull(STDERR_FILENO, buf, count); + } } @@ -317,24 +319,50 @@ static void performOp(Source & from, Sink & to, unsigned int op) writeStringSet(paths, to); break; } + + case wopQueryDeriver: { + Path path = readStorePath(from); + startWork(); + Path deriver = store->queryDeriver(path); + stopWork(); + writeString(deriver, to); + break; + } + + case wopQueryDerivers: { + Path path = readStorePath(from); + string identifier = readString(from); + string user = readString(from); + startWork(); + PathSet derivers = store->queryDerivers(path, identifier, user); + stopWork(); + writeStringSet(derivers, to); + break; + } case wopAddToStore: { - /* !!! uberquick hack */ string baseName = readString(from); bool fixed = readInt(from) == 1; bool recursive = readInt(from) == 1; string hashAlgo = readString(from); - + + printMsg(lvlInfo, format("NIXWORKER: WOP 1")); + + /* !!! uberquick hack */ Path tmp = createTempDir(); AutoDelete delTmp(tmp); Path tmp2 = tmp + "/" + baseName; restorePath(tmp2, from); + printMsg(lvlInfo, format("NIXWORKER: WOP 2")); + startWork(); Path path = store->addToStore(tmp2, fixed, recursive, hashAlgo); stopWork(); - + + printMsg(lvlInfo, format("NIXWORKER: WOP 3 '%1%'") % path); writeString(path, to); + printMsg(lvlInfo, format("NIXWORKER: WOP 4")); break; } @@ -468,7 +496,8 @@ static void processConnection() /* Exchange the greeting. */ unsigned int magic = readInt(from); - if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); + if (magic != WORKER_MAGIC_1) + throw Error("protocol mismatch"); verbosity = (Verbosity) readInt(from); writeInt(WORKER_MAGIC_2, to); @@ -506,7 +535,9 @@ static void processConnection() opCount++; try { + printMsg(lvlInfo, format("Processing op '%1%'") % op); performOp(from, to, op); + printMsg(lvlInfo, format("Processed op '%1%'") % op); } catch (Error & e) { stopWork(false, e.msg()); } @@ -535,6 +566,20 @@ static void setSigChldAction(bool autoReap) throw SysError("setting SIGCHLD handler"); } +/* Returns the user ID of the one connecting to the socket though + * SCM_CREDENTIALS. This uid is checked by the kernel and cannot be faked :) + */ +uid_t getpeereuid(int sd) +{ + struct ucred cred; + int len = sizeof (cred); + socklen_t len2 = (socklen_t)len; + + if ( getsockopt(sd, SOL_SOCKET, SO_PEERCRED, &cred, &len2) ) + throw SysError("Cannot get SCM_CREDENTIALS for socket"); + + return cred.uid; +} static void daemonLoop() { @@ -542,6 +587,7 @@ static void daemonLoop() zombies. */ setSigChldAction(true); + /* Create and bind to a Unix domain socket. */ AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); if (fdSocket == -1) @@ -575,7 +621,8 @@ static void daemonLoop() /* Important: the server process *cannot* open the Berkeley DB environment, because it doesn't like forks very much. */ - assert(!store); + if(!debugWorker) + assert(!store); /* Accept a connection. */ struct sockaddr_un remoteAddr; @@ -585,42 +632,76 @@ static void daemonLoop() (struct sockaddr *) &remoteAddr, &remoteAddrLen); checkInterrupt(); if (remote == -1) - if (errno == EINTR) - continue; - else - throw SysError("accepting connection"); + if (errno == EINTR) + continue; + else + throw SysError("accepting connection"); - printMsg(lvlInfo, format("accepted connection %1%") % remote); + uid_t caller_uid = getpeereuid(remote); - /* Fork a child to handle the connection. */ - pid_t child; - child = fork(); - - switch (child) { - - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - /* Background the worker. */ - if (setsid() == -1) - throw SysError(format("creating a new session")); - - /* Restore normal handling of SIGCHLD. */ - setSigChldAction(false); - - /* Handle the connection. */ - from.fd = remote; - to.fd = remote; - processConnection(); - - } catch (std::exception & e) { - std::cerr << format("child error: %1%\n") % e.what(); - } - exit(0); - } + printMsg(lvlInfo, format("accepted connection '%1%'") + % remote); + + if(!debugWorker) + { + + /* Fork a child to handle the connection. */ + pid_t child; + child = fork(); + + switch (child) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + /* We set the user id of the caller in the fork */ + setCallingUID(caller_uid); + printMsg(lvlInfo, format("Fork for connection '%1%' created for userid: '%2%' with username: '%3%'") + % remote % queryCallingUID() % queryCallingUsername()); + + /* Background the worker. */ + if (setsid() == -1) + throw SysError(format("creating a new session")); + + /* Restore normal handling of SIGCHLD. */ + setSigChldAction(false); + + /* Handle the connection. */ + from.fd = remote; + to.fd = remote; + processConnection(); + + } catch (std::exception & e) { + std::cerr << format("child error: %1%\n") % e.what(); + } + exit(0); + } + } + else + { + printMsg(lvlInfo, format("Nix-worker: debug mode: we can only process 1 job at the time")); + + /* We RESET the user id of the caller cause we have no fork */ + setCallingUID(caller_uid, true); + printMsg(lvlInfo, format("Debug connection '%1%' created for userid: '%2%' with username: '%3%'") + % remote % queryCallingUID() % queryCallingUsername()); + + /* Background the worker. */ + //if (setsid() == -1) + // throw SysError(format("creating a new session")); + + /* Restore normal handling of SIGCHLD. */ + //setSigChldAction(false); + + /* Handle the connection. */ + from.fd = remote; + to.fd = remote; + processConnection(); + + } } catch (Interrupted & e) { throw;