From 79d5604780dabb58d2d2f5ca297e6d87eee9da7e Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Thu, 7 Jun 2007 13:16:38 +0000 Subject: [PATCH] Changed commit script: it recursively walkes through all dirs itself now, uses svn stat where needed, and doesnt use svn add *,svn revert anymore and is much faster --- scripts/nix-statecommit.sh.in | 160 +++++++++++++++++++++------------- src/libexpr/primops.cc | 14 +-- src/libstore/build.cc | 4 +- src/libstore/local-store.cc | 97 +++++++++++++++------ src/libstore/local-store.hh | 6 +- src/libstore/remote-store.cc | 13 ++- src/libstore/remote-store.hh | 6 +- src/libstore/store-api.hh | 8 +- src/libstore/store-state.cc | 7 +- src/libutil/util.cc | 2 +- src/nix-state/nix-state.cc | 25 ++++-- 11 files changed, 229 insertions(+), 113 deletions(-) diff --git a/scripts/nix-statecommit.sh.in b/scripts/nix-statecommit.sh.in index 5974fc80c..449c21683 100755 --- a/scripts/nix-statecommit.sh.in +++ b/scripts/nix-statecommit.sh.in @@ -1,5 +1,5 @@ #! /bin/sh -# we cant do a -e since ... +# we cant do a -e since ... svn can fail ??? TODO... #check if there are enough arguments, if not, exit with an error @@ -17,21 +17,98 @@ nonversionedpaths=( $4 ) checkouts=( $5 ) #echo svnbin: $svnbin -#echo subversionedpaths: $subversionedpaths -#echo subversionedpathsCommitBools: $subversionedpathsCommitBools -#echo nonversionedpaths: $nonversionedpaths -#echo checkouts: $checkouts +#echo subversionedpaths: ${subversionedpaths[@]} +#echo subversionedpathsCommitBools: ${subversionedpathsCommitBools[@]} +#echo nonversionedpaths: ${nonversionedpaths[@]} +#echo checkouts: ${checkouts[@]} - - -#TODO # -# silence "is already under version control" messages -# after a "revert $x" you can silence the "skipped '$x'" # -#TODO +# +# +function subversionSingleStateDir { + excludelist=( "." ".." ".svn" ); + cd $1; + echo cd $1; + + empty=$(ls) + + if [ "$empty" = "" ] ; then + allsubitems=(); #no subfiles / dirs + else + allsubitems=( $(echo *) $(echo .*) ) #there are subfiles / dirs,also adds hidden items + fi + + for subitem in ${allsubitems[@]} + do + if [ "$subitem" = ".svn" ]; then + allsubitems=( $(svn -N stat | sed -n '/^?/p' | sed 's/? //' | tr -d "\12") ) #there are subfiles, and theyre already versioned + fi + done + + #echo "Allsubitems ${allsubitems[@]}" + + subitems=(); + for subitem in ${allsubitems[@]} #add all, were going to exlucde explicity stated direct versioned-subdirs or explicity stated nonversioned-subdirs + do #this is only to prevent some warnings, ultimately we would like svn add to have a option 'exclude dirs' + + exclude=0; + + for excl in ${excludelist[@]} #check if the subitem is in the list of excluded dirs + do + if [ "$excl" = "$subitem" ]; then + exclude=1; + #echo "exclude $excl" + fi + done + + subitem="$(pwd)/$subitem"; #create the full path + + if test -d $subitem; then #the subitem (file or a dir) may be a dir, so we add a / to the end + subitem="$subitem/"; + fi + + for svnp in ${subversionedpaths[@]} #check if the subitem is in the list of subverioned paths + do + if [ "$svnp" = "$subitem" ]; then + exclude=1; + #echo "exclude versioned $svnp" + fi + done + + for nonvp in ${nonversionedpaths[@]} #check if the subitem is in the list of dirs that aren't supposed to be versioned + do + if [ "$nonvp" = "$subitem" ]; then + exclude=1; + #echo "exclude nonversioned $svnp" + fi + done + + if [ $exclude = 0 ]; then #Exclude the subitem if nessecary + subitems[${#subitems[*]}]=$subitem + fi + done + + #echo ${subitems[@]} + + for item in ${subitems[@]} + do + if test -d $item; then #add or go recursive subitems + $debug svn -N add $item #NON recursively add the dir + subversionSingleStateDir $item + else + $debug svn add $item + fi + done + +} + +# +# +# +# i=0 for path in ${subversionedpaths[@]} @@ -40,65 +117,18 @@ do cd $path; output=$($svnbin stat 2>&1 | grep "is not a working copy"); - if [ "$output" != "" ] ; then #if the dir exists but is not yet an svn dir: create repos, if it doenst exits (is removed or something) than we dont do anything + if [ "$output" != "" ] ; then #if the dir exists but is not yet an svn dir: checkout repos, if it doenst exits (is removed or something) than we dont do anything $debug ${checkouts[$i]}; fi if [ "${subversionedpathsCommitBools[$i]}" = "true" ]; then #Check if we need to commit this folder echo "Entering $path" - - allsubitems=( $(echo *) ) #TODO, maybe also add hidden files starting with a '.' , but we dont want to add .svn dirs - #TODO2 maybe do something with svn stat to speed up ? - subitems=(); - for subitem in ${allsubitems[@]} #add all, were going to exlucde explicity stated direct versioned-subdirs or explicity stated nonversioned-subdirs - do #this is only to prevent some warnings, ultimately we would like svn add to have a option 'exclude dirs' - subitem="$(pwd)/$subitem"; - - if test -d $subitem; then #the subitem (file or a dir) may be a dir, so we add a / to the end - subitem="$subitem/"; - fi - - exclude=0; - - for svnp in ${subversionedpaths[@]} #check if the subitem is in the list of subverioned paths - do - if [ "$svnp" = "$subitem" ]; then - exclude=1; - #echo "exclude versioned $svnp" - fi - done - - for nonvp in ${nonversionedpaths[@]} #check if the subitem is in the list of dirs that aren't supposed to be versioned - do - if [ "$nonvp" = "$subitem" ]; then - exclude=1; - #echo "exclude nonversioned $svnp" - fi - done - - if [ $exclude = 0 ]; then #Exclude the subitem if nessecary - subitems[${#subitems[*]}]=$subitem - fi - done + + subversionSingleStateDir $path; - if [ "$subitems" != "" ]; then - echo "adding ${subitems[@]}" - $debug svn add ${subitems[@]} #add all subitems - - for revpath in ${nonversionedpaths[@]} #We need to revert sub-sub* dirs, since these havent been excluded - do - if test -d $revpath; then - if [ "${revpath:0:${#path}}" == "$path" ]; then - echo "Revert $revpath"; - $debug svn revert $revpath; - fi - fi - done - - $debug svn -m "" commit; #Finally, we commit - fi - + cd $path #now that everything is added we can commit + $debug svn -m "" commit; fi cd - &> /dev/null; @@ -106,3 +136,7 @@ do let "i+=1" done + + + + diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 65b766796..71183d182 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -470,7 +470,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args) s = s + "/"; //Remove the / at the beginning if it's there - if(s[0] == '/') + if(s[0] == '/' && s.length() != 1) s = s.substr(1, s.length()); if(s == stateRootRepos) @@ -619,13 +619,17 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args) printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'") % drvName % drvPath); - /* TODO Write updated (no need to rebuild) state derivations to a special table, so they can be updated at build time */ + /* Write updated (no need to rebuild) state derivations to the database, so they can be updated at build time */ if(enableState && !disableState){ - Path deriver = queryDeriver(noTxn, outPath); //query deriver - if(deriver != drvPath){ - store->setUpdatedStateDerivation(drvPath, outPath); + if(store->isValidPath(outPath)){ //Only add when the path is already valid + Path deriver = queryDeriver(noTxn, outPath); //query the deriver + if(deriver != drvPath){ + store->addUpdatedStateDerivation(drvPath, outPath); + } } + //TODO Also add when path is not already valid, which drv does it take at build time, the latest ... guess so .. so we dont need to add? } + /* Optimisation, but required in read-only mode! because in that case we don't actually write store expressions, so we can't diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2d978915b..848d8d552 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2477,12 +2477,14 @@ void LocalStore::buildDerivations(const PathSet & drvPaths) startNest(nest, lvlDebug, format("building %1%") % showPaths(drvPaths)); + //Just before we build, we resolve the multiple derivations linked to one store path issue, by choosing the latest derivation + store->updateAllStateDerivations(); + Worker worker; Goals goals; for (PathSet::const_iterator i = drvPaths.begin(); i != drvPaths.end(); ++i){ goals.insert(worker.makeDerivationGoal(*i)); - printMsg(lvlError, format("No component build, but state check: %1%") % *i); } worker.run(goals); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 08e2fad67..77a821953 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1223,41 +1223,82 @@ PathSet LocalStore::getStateReferencesClosure(const Path & path) } -//TODO -void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath) +void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath) { Transaction txn(nixDB); + //Check wheter were not duplicating an entry, if so, than we wont have to do anything + //TODO + if(false){ + + return; + } + Strings data; data.push_back(storepath); - time_t timestamp; time (×tamp); string timestamp_s = time_t2string(timestamp); printMsg(lvlError, format("Adding new drv (%1%) to replace drv of old component (%2%) with timestamp: %3%") % newdrv % storepath % timestamp_s); data.push_back(timestamp_s); //create a timestamp to remember which one was last inserted - nixDB.setStrings(txn, dbUpdatedDerivations, newdrv, data); - //TODO check wheter were not duplicating an entry !! - txn.commit(); } -//TODO -void LocalStore::setUpdatedStateDerivation(const Path & newdrv, const Path & storepath) +void LocalStore::addUpdatedStateDerivation(const Path & newdrv, const Path & storepath) { - nix::setUpdatedStateDerivation(newdrv, storepath); + nix::addUpdatedStateDerivation(newdrv, storepath); } -//TODO -Path updateStateDerivation(const Path & storepath) +void updateAllStateDerivations() { - Transaction txn(nixDB); - - Path drvPath = queryDeriver(txn, storepath); - printMsg(lvlError, format("Current DRV: %1%") % drvPath); + Transaction txn(nixDB); + Strings unique_paths; + + Strings keys; + nixDB.enumTable(txn, dbUpdatedDerivations, keys); + for (Strings::iterator i = keys.begin(); i != keys.end(); ++i) + { + string drv_key = *i; //the key is the derivation + Strings data; + nixDB.queryStrings(txn, dbUpdatedDerivations, drv_key, data); + string path = data.front(); + bool exists = false; + + for (Strings::iterator j = unique_paths.begin(); j != unique_paths.end(); ++j) + { + string unique_path = *j; + + if(path == unique_path) + exists = true; + } + + if(!exists) + unique_paths.push_back(path); + } + + for (Strings::iterator i = unique_paths.begin(); i != unique_paths.end(); ++i) + { + string path = *i; + printMsg(lvlError, format("Unique: %1%") % path); + store->updateStateDerivation(txn, path); //TODO replace store-> + } + + txn.commit(); +} + +void LocalStore::updateAllStateDerivations() +{ + nix::updateAllStateDerivations(); +} + + +void updateStateDerivation(const Transaction & txn, const Path & storepath) +{ + Path drvPath = queryDeriver(txn, storepath); + Path originalDerivation = drvPath; Path newDerivation = drvPath; //the new drv path first equals the old one until a new one is found int timestamp = 0; @@ -1280,30 +1321,34 @@ Path updateStateDerivation(const Path & storepath) string2Int(gettimestamp, gettimestamp_i); if(gettimestamp_i == timestamp) - throw Error(format("Error! Multiple changes at the same time of derivation: `%1%' with timestamp") % drv_key); //TODO delete the mulitple changes ?? + throw Error(format("Error! Multiple changes at store path %4% at the same time: derivations: `%1%' and `%2%' with timestamp `%3%'") % newDerivation % drv_key % timestamp % storepath); if(timestamp == 0 || gettimestamp_i > timestamp){ //we choose the new derivation as the latest submitted derivation - printMsg(lvlError, format("Replacing old drv (%1%) with new drv (%2%) with timestamp: %3%") % newDerivation % drv_key % gettimestamp_i); + //printMsg(lvlError, format("Replacing at store path %4% the old drv (%1%) with new drv (%2%) with timestamp: %3%") % newDerivation % drv_key % gettimestamp_i % storepath); newDerivation = drv_key; timestamp = gettimestamp_i; - - //Replace the old deriver link in the derivers database (TODO, and delete old deriver path????????) - setDeriver(txn, storepath, newDerivation); } //Always Remove the old updatelink in the dbUpdatedDerivations nixDB.delPair(txn, dbUpdatedDerivations, drv_key); } } - - txn.commit(); - return newDerivation; + + if(originalDerivation != newDerivation) //only update if neccecary + { + //Replace the old deriver link in the derivers database (TODO, maybe delete old deriver path????????) + setDeriver(txn, storepath, newDerivation); + + //Call the stateUpdate function for the new derivation? Yes since this function is called at build time + printMsg(lvlError, format("Calling new state drv %1% for storepath %2%") % newDerivation % storepath); + //TODO check wheter we update before or after ?? + //TODO + } } -//TODO -Path LocalStore::updateStateDerivation(const Path & storepath) +void LocalStore::updateStateDerivation(const Transaction & txn, const Path & storepath) { - return nix::updateStateDerivation(storepath); + nix::updateStateDerivation(txn, storepath); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 86d66cd52..c5d1cbadd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -84,9 +84,11 @@ public: PathSet getStateReferencesClosure(const Path & path); - void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath); + void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath); - Path updateStateDerivation(const Path & storepath); + void updateStateDerivation(const Transaction & txn, const Path & storepath); + + void updateAllStateDerivations(); }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index dd8d29f6f..b89970788 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -403,16 +403,21 @@ PathSet RemoteStore::getStateReferencesClosure(const Path & path) } //TODO -void RemoteStore::setUpdatedStateDerivation(const Path & newdrv, const Path & storepath) +void RemoteStore::addUpdatedStateDerivation(const Path & newdrv, const Path & storepath) { } //TODO -Path RemoteStore::updateStateDerivation(const Path & storepath) +void RemoteStore::updateStateDerivation(const Transaction & txn, const Path & storepath) { - Path p; - return p; + +} + +//TODO +void RemoteStore::updateAllStateDerivations() +{ + } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9cea22f09..86c9937d4 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -72,9 +72,11 @@ public: PathSet getStateReferencesClosure(const Path & path); - void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath); + void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath); - Path updateStateDerivation(const Path & storepath); + void updateStateDerivation(const Transaction & txn, const Path & storepath); + + void updateAllStateDerivations(); private: diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c810778ca..af63e5194 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -9,6 +9,7 @@ #include "hash.hh" #include "serialise.hh" #include "derivations.hh" +#include "db.hh" namespace nix { @@ -194,10 +195,13 @@ public: virtual PathSet getStateReferencesClosure(const Path & path) = 0; /* TODO */ - virtual void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath) = 0; + virtual void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath) = 0; /* TODO */ - virtual Path updateStateDerivation(const Path & storepath) = 0; + virtual void updateStateDerivation(const Transaction & txn, const Path & storepath) = 0; + + /* TODO */ + virtual void updateAllStateDerivations() = 0; }; diff --git a/src/libstore/store-state.cc b/src/libstore/store-state.cc index 38bd36f6b..28248fba9 100644 --- a/src/libstore/store-state.cc +++ b/src/libstore/store-state.cc @@ -16,9 +16,12 @@ namespace nix { -void dsfsdfas() +void updatedStateDerivation(Path storePath) { - + //Remove the old .svn folders + + //Create new repositorys, or use existing... + //createStateDirs already does that ... } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a2e5810f7..a41125a54 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1016,7 +1016,7 @@ string trim(const string & s) { void executeAndPrintShellCommand(const string & command, const string & commandName) { - string tempoutput = "svnoutput.txt"; + string tempoutput = "/tmp/svnoutput.txt"; string newcommand = command + " &> " + tempoutput; //the &> sends also stderr to stdout int kidstatus, deadpid; diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index 6ce035542..9375441f6 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -102,7 +102,9 @@ static void opRunComponent(Strings opFlags, Strings opArgs) //Check if component is a state component !!! - //Wait for locks? + //Check for locks ... + //add locks ... ? + //svn lock ... ? //******************* Run the component //TODO @@ -156,7 +158,10 @@ static void opRunComponent(Strings opFlags, Strings opArgs) DerivationStateOutputDir d = i->second; string thisdir = d.path; + string fullstatedir = statePath + "/" + thisdir; + if(thisdir == "/") //exception for the root dir + fullstatedir = statePath + "/"; Path statePath = fullstatedir; //TODO call coerce function if(d.type == "none"){ @@ -229,11 +234,13 @@ void run(Strings args) Strings opFlags, opArgs; Operation op = 0; - /* test */ + /* test * store = openStore(); - Path p = "/nix/store/l569q3a2cfx834mcf3vhwczjgbaljnp7-hellohardcodedstateworld-1.0"; - store->setUpdatedStateDerivation("/nix/store/63xcbrk3v5nbn9qla7rwnx6rvz3iqm5l-hellohardcodedstateworld-1.0.drv", p); - store->updateStateDerivation(p); + Path p = "/nix/store/l569q3a2cfx834mcf3vhwczjgbaljnp7-hellohardcodedstateworld-1.0"; // + store->addUpdatedStateDerivation("/nix/store/63xcbrk3v5nbn9qla7rwnx6rvz3iqm5l-hellohardcodedstateworld-1.0.drv", p); // + Path p2 = "/nix/store/4ycq45hsgc8yaj4vwafx3lgd473jaqwg-hellohardcodedstateworld-1.0"; + store->addUpdatedStateDerivation("/nix/store/s6wggk924jx0gcb0l29ra4g9fxa3b4pp-hellohardcodedstateworld-1.0.drv", p2); // + store->updateAllStateDerivations(); return; /* test */ @@ -261,6 +268,14 @@ void run(Strings args) TODO update getDerivation in nix-store to handle state indentifiers + --update state drv + + --revert-to-state (recursive revert...) + + --delete state? + + -- + */ else