From 267ccc589dc2dcefff346385d3375724ed499c1e Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Fri, 8 Jun 2007 16:00:55 +0000 Subject: [PATCH] Nix now understands the difference between runtime-state-components and non-runtime-state-compontens. Components and Derivations are now properly (re)build/derived (or not) when necessary. --- scripts/nix-statecommit.sh.in | 42 ++++++++++++------------ src/libexpr/primops.cc | 61 +++++++++++++++++++++++++---------- src/libstore/derivations.hh | 29 ++++++++++++++--- src/libstore/local-store.cc | 2 +- src/libstore/store-state.cc | 37 +++++++++------------ src/libutil/util.cc | 21 ++++++++++++ src/libutil/util.hh | 5 +++ src/nix-state/nix-state.cc | 4 +-- 8 files changed, 135 insertions(+), 66 deletions(-) diff --git a/scripts/nix-statecommit.sh.in b/scripts/nix-statecommit.sh.in index aca15bf0d..528169e96 100755 --- a/scripts/nix-statecommit.sh.in +++ b/scripts/nix-statecommit.sh.in @@ -5,7 +5,7 @@ debug=""; #set to "" for no debugging, set to "echo " to debug the commands -if [ "$#" != 5 ] || [ "$#" != 6 ] ; then +if [ "$#" != 5 ] && [ "$#" != 6 ] ; then echo "Incorrect number of arguments" exit 1; fi @@ -22,11 +22,12 @@ nonversionedpaths=( $4 ) checkouts=( $5 ) deletesvn=$6 #this flag can be set to 1 to DELETE all .svn folders and NOT commit -#echo svnbin: $svnbin -#echo subversionedpaths: ${subversionedpaths[@]} -#echo subversionedpathsCommitBools: ${subversionedpathsCommitBools[@]} -#echo nonversionedpaths: ${nonversionedpaths[@]} -#echo checkouts: ${checkouts[@]} +echo svnbin: $svnbin +echo subversionedpaths: ${subversionedpaths[@]} +echo subversionedpathsCommitBools: ${subversionedpathsCommitBools[@]} +echo nonversionedpaths: ${nonversionedpaths[@]} +echo checkouts: ${checkouts[@]} +echo deletesvn: $deletesvn # # @@ -125,23 +126,23 @@ function subversionSingleStateDir { # i=0 +i_checkout=0 for path in ${subversionedpaths[@]} do if test -d $path; then #if the dir doesnt exist, than we dont hav to do anything cd $path; - - #HACK: I cant seem to find a way for bash to parse a 2 dimensional string array as argument, so we use a 1-d array with '|' as seperator - if ! test -d $path/.svn/; then #if the dir exists but is not yet an svn dir: checkout repos, if it doenst exits (is removed or something) than we dont do anything - checkoutcommand=""; - while true; do - if [ "${checkouts[$i]}" = "|" ]; then - break - let "i+=1" - fi - checkoutcommand="${checkoutcommand} ${checkouts[$i]}"; - let "i+=1" - done - + + checkoutcommand=""; #HACK: I cant seem to find a way for bash to parse a 2 dimensional string array as argument, so we use a 1-d array with '|' as seperator + while true; do + if [ "${checkouts[$i_checkout]}" = "|" ]; then + let "i_checkout+=1" + break + fi + checkoutcommand="${checkoutcommand} ${checkouts[$i_checkout]}"; + let "i_checkout+=1" + done + + if ! test -d "${path}.svn/"; then #if the dir exists but is not yet an svn dir: checkout repos, if it doenst exits (is removed or something) than we dont do anything if [ "$deletesvn" != "1" ]; then $debug $checkoutcommand; fi @@ -157,9 +158,10 @@ do if [ "$deletesvn" != "1" ]; then $debug svn -m "" commit; fi - fi + fi cd - &> /dev/null; + let "i+=1" fi done diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fe3e7be31..b6d4a55d2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -317,14 +317,16 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv) { //printMsg(lvlError, format("DRV: %1% %2%") % drv.env.find("name")->second % (drv.outputs.size() == 1)); - /* Return a fixed hash for fixed-output derivations. */ + /* Return a fixed hash for fixed-output derivations. + * E.g. derivations that have a hash + */ if (drv.outputs.size() == 1) { DerivationOutputs::const_iterator i = drv.outputs.begin(); if (i->first == "out" && i->second.hash != "") { - //printMsg(lvlError, format("%1% - %2% - %3%") % i->second.hashAlgo % i->second.hash % i->second.path); + //printMsg(lvlError, format("FIXED OUTPUT: %1% - %2% - %3%") % i->second.hashAlgo % i->second.hash % i->second.path); return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" @@ -333,18 +335,30 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv) } } - /* If we have a state derivation, we clear these paramters because they dont affect to outPath */ + /* If we have a state derivation, we clear state paramters because they (sometimes) dont affect to outPath: + * If this drv has runtime paramters: The state indentifier and thus statepath may change, but the componentPath (outPath) can stay the same + * If this drv doesnt have runtime paramters: The state indentifier and thus statepath may change, and thus the componentPath changes since it is build with another identifier + * In both cases: Other runtime state parameters like stateDirs, synchronisation and shareState never change the out or statepath so always need to be out of the hash + */ if(drv.stateOutputs.size() != 0){ - drv.env["statepath"] = ""; - drv.stateOutputs.clear(); - drv.stateOutputDirs.clear(); - - /* We do NOT clear the state identifier (what about the username ????) when there are NO - * runtime arguments, since this will affect the statePath and therefore the outPath - */ - //TODO - } - + + if(drv.stateOutputs.size() != 1) + throw EvalError(format("There are more then one stateOutputs in the derviation.....")); + + DerivationStateOutput drvso = drv.stateOutputs["state"]; + + if(drvso.runtimeStateParamters != ""){ //Has runtime parameters --> Clear all state parameters + drv.stateOutputs.clear(); + drv.stateOutputDirs.clear(); + drv.env["statepath"] = ""; + printMsg(lvlError, format("YES RUNTIME")); + } + else{ //Has NO runtime parameters --> Clear state parameters selectively + drvso.clearAllRuntimeParamters(); + drv.stateOutputDirs.clear(); + printMsg(lvlError, format("NO RUNTIME")); + } + } /* For other derivations, replace the inputs paths with recursive calls to this function.*/ @@ -360,9 +374,7 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv) inputs2[printHash(h)] = i->second; } drv.inputDrvs = inputs2; - - //printMsg(lvlError, format("%1%") % unparseDerivation(drv)); - + //printMsg(lvlError, format("%1% with %2% --> %3%") % drv.env.find("name")->second % printHash(hashTerm(unparseDerivation(drv))) % unparseDerivation(drv)); return hashTerm(unparseDerivation(drv)); } @@ -583,10 +595,25 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args) output names do get reflected in the hash. */ drv.env["out"] = ""; drv.outputs["out"] = DerivationOutput("", outputHashAlgo, outputHash); + + /* If there are no runtime paratermers, then we need to take the the stateIdentifier into account for the hash calcaulation below: hashDerivationModulo(...) + * We also add enableState to make it parse the drv to a state-drv + * We also add runtimeStateParamters for the hash calc in hashDerivationModulo(...) to check if its needs to take the stateIdentiefier into account in the hash + */ + if(enableState && !disableState){ + if(runtimeStateParamters == ""){ + string enableStateS = bool2string("true"); + drv.stateOutputs["state"] = DerivationStateOutput("", "", "", stateIdentifier, enableStateS, "", "", "", runtimeStateParamters, false); + } + } /* Use the masked derivation expression to compute the output path. */ - Path outPath = makeStorePath("output:out", hashDerivationModulo(state, drv), drvName); + + //TODO CLEANUP + Hash h = hashDerivationModulo(state, drv); + //printMsg(lvlError, format("USES: `%1%'") % printHash(h)); + Path outPath = makeStorePath("output:out", h, drvName); /* Construct the final derivation store expression. */ drv.env["out"] = outPath; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 617e20e83..7cbee8938 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -49,15 +49,19 @@ struct DerivationStateOutput string createDirsBeforeInstall; //if true: creates state dirs before installation string runtimeStateParamters; //if not empty: these are the runtime parameters where state can be found (you can use $statepath here) + DerivationStateOutput() { } - DerivationStateOutput(Path statepath, string hashAlgo, string hash, string stateIdentifier, string enabled, string shared, string synchronization, string createDirsBeforeInstall, string runtimeStateParamters) + + DerivationStateOutput(Path statepath, string hashAlgo, string hash, string stateIdentifier, string enabled, string shared, string synchronization, string createDirsBeforeInstall, string runtimeStateParamters, bool check=true) { - if(shared != "none" && shared != "full" && shared != "group") - throw Error(format("shared '%1%' is not a correct type") % shared); - if(synchronization != "none" && synchronization != "exclusive-lock" && synchronization != "recursive-exclusive-lock") - throw Error(format("synchronization '%1%' is not a correct type") % synchronization); + if(check){ + if(shared != "none" && shared != "full" && shared != "group") + throw Error(format("shared '%1%' is not a correct type") % shared); + if(synchronization != "none" && synchronization != "exclusive-lock" && synchronization != "recursive-exclusive-lock") + throw Error(format("synchronization '%1%' is not a correct type") % synchronization); + } //TODO //commitReferences @@ -81,6 +85,21 @@ struct DerivationStateOutput bool getCreateDirsBeforeInstall(){ return string2bool(createDirsBeforeInstall); } + + /* + * This sets all the paramters to "" to ensure they're not taken into account for the hash calculation in primops.cc + */ + void clearAllRuntimeParamters(){ + this->statepath = ""; + //this->hashAlgo; //Clear this one? + //this->hash; //Clear this one? + //this->stateIdentifier; + this->enabled = ""; + this->shared = ""; + this->synchronization = ""; + this->createDirsBeforeInstall = ""; + this->runtimeStateParamters = ""; + } }; struct DerivationStateOutputDir diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 77a821953..4c0109e1c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1282,7 +1282,7 @@ void updateAllStateDerivations() for (Strings::iterator i = unique_paths.begin(); i != unique_paths.end(); ++i) { string path = *i; - printMsg(lvlError, format("Unique: %1%") % path); + //printMsg(lvlError, format("Unique: %1%") % path); store->updateStateDerivation(txn, path); //TODO replace store-> } diff --git a/src/libstore/store-state.cc b/src/libstore/store-state.cc index fca0aa76c..3da147b5f 100644 --- a/src/libstore/store-state.cc +++ b/src/libstore/store-state.cc @@ -18,21 +18,14 @@ namespace nix { void updatedStateDerivation(Path storePath) { - //Remove the old .svn folders + //We dont remove the old .svn folders + //New repostorys are created by createStateDirs - //Create new repositorys, or use existing... - //createStateDirs already does that ... printMsg(lvlError, format("Resetting state drv settings like repositorys")); //Create a repository for this state location - //string repos = makeStateReposPath("stateOutput:staterepospath", stateDir, thisdir, drvName, stateIdentifier); - //executeAndPrintShellCommand("mkdir -p " + repos, "mkdir"); - //executeAndPrintShellCommand(svnadminbin + " create " + repos, "svnadmin"); //TODO create as nixbld.nixbld chmod 700... can you still commit than ?? - - - //createStateDirs - + // } void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const DerivationStateOutputs & stateOutputs, const StringPairs & env) @@ -47,7 +40,7 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De PathSet intervalPaths; - //TODO check if we can create stata and staterepos dirs + //TODO check if we can create state and staterepos dirs for (DerivationStateOutputDirs::const_reverse_iterator i = stateOutputDirs.rbegin(); i != stateOutputDirs.rend(); ++i){ DerivationStateOutputDir d = i->second; @@ -56,31 +49,33 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De string fullstatedir = stateDir + "/" + thisdir; Path statePath = fullstatedir; //TODO call coerce function - //TODO REPLACE TRUE INTO VAR OF CREATEING DIRS BEFORE OR AFTER INSTALL //Check if and how this dir needs to be versioned if(d.type == "none"){ - if(true){ - executeAndPrintShellCommand("mkdir -p " + fullstatedir, "mkdir"); - } + executeAndPrintShellCommand("mkdir -p " + fullstatedir, "mkdir"); continue; } //Create a repository for this state location string repos = makeStateReposPath("stateOutput:staterepospath", stateDir, thisdir, drvName, stateIdentifier); executeAndPrintShellCommand("mkdir -p " + repos, "mkdir"); - executeAndPrintShellCommand(svnadminbin + " create " + repos, "svnadmin"); //TODO create as nixbld.nixbld chmod 700... can you still commit than ?? -// //TODO Check if repos already exitst? + + if(IsDirectory(repos)) + executeAndPrintShellCommand(svnadminbin + " create " + repos, "svnadmin"); //TODO create as nixbld.nixbld chmod 700... can you still commit than ?? + else + printMsg(lvlError, format("Repos %1% already exists, so we use that repository") % repos); if(d.type == "interval"){ intervalPaths.insert(statePath); } - //TODO REPLACE TRUE INTO VAR OF CREATEING DIRS BEFORE OR AFTER INSTALL - if(true){ - printMsg(lvlError, format("Adding state subdir: %1% to %2% from repository %3%") % thisdir % fullstatedir % repos); + printMsg(lvlError, format("Adding state subdir: %1% to %2% from repository %3%") % thisdir % fullstatedir % repos); + + if(IsDirectory(fullstatedir + "/.svn/")){ string checkoutcommand = svnbin + " checkout file://" + repos + " " + fullstatedir; - executeAndPrintShellCommand(checkoutcommand, "svn"); //TODO checkout as user + executeAndPrintShellCommand(checkoutcommand, "svn"); //TODO checkout as user } + else + printMsg(lvlError, format("Statedir %1% already exists, so dont check out its repository again") % fullstatedir); } //Initialize the counters for the statePaths that have an interval to 0 diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a41125a54..e8f12e7a3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -18,6 +18,7 @@ #include #include + extern char * * environ; @@ -1071,5 +1072,25 @@ string time_t2string(const time_t & t) string s = int2String(i); return s; } + +bool FileExist(const string FileName) +{ + const char* FileName_C = FileName.c_str(); + //strcpy(FileName_C, FileName.c_str()); + + struct stat my_stat; + return (stat(FileName_C, &my_stat) == 0); +} + +bool IsDirectory(const string FileName) +{ + const char* FileName_C = FileName.c_str(); + //strcpy(FileName_C, FileName.c_str()); + + struct stat my_stat; + if (stat(FileName_C, &my_stat) != 0) return false; + return ((my_stat.st_mode & S_IFDIR) != 0); +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index cbbf16ef5..68e1e798a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -296,6 +296,11 @@ void executeAndPrintShellCommand(const string & command, const string & commandN //Convert time_t to a string string time_t2string(const time_t & t); +bool FileExist(const string FileName); + +bool IsDirectory(const string FileName); + + } #endif /* !__UTIL_H */ diff --git a/src/nix-state/nix-state.cc b/src/nix-state/nix-state.cc index f150e78eb..63fea3093 100644 --- a/src/nix-state/nix-state.cc +++ b/src/nix-state/nix-state.cc @@ -189,7 +189,7 @@ static void opRunComponent(Strings opFlags, Strings opArgs) string repos = makeStateReposPath("stateOutput:staterepospath", statePath, thisdir, drvName, stateIdentifier); //this is a copy from store-state.cc // - checkoutcommands.push_back(svnbin + " checkout file://" + repos + " " + fullstatedir); + checkoutcommands.push_back(svnbin + " --ignore-externals checkout file://" + repos + " " + fullstatedir); subversionedpaths.push_back(fullstatedir); if(d.type == "interval"){ @@ -260,7 +260,7 @@ void run(Strings args) store->updateAllStateDerivations(); return; /* test */ - + for (Strings::iterator i = args.begin(); i != args.end(); ) { string arg = *i++;