mirror of
https://github.com/NixOS/nix.git
synced 2025-11-26 04:00:59 +01:00
Changed commit script: it recursively walkes through all dirs itself now, uses svn stat where needed, and doesnt use svn add *,svn revert anymore and is much faster
This commit is contained in:
Wouter den Breejen
parent
7166ad8eba
commit
79d5604780
11 changed files with 229 additions and 113 deletions
|
|
@ -1,5 +1,5 @@
|
|||
#! /bin/sh
|
||||
# we cant do a -e since ...
|
||||
# we cant do a -e since ... svn can fail ??? TODO...
|
||||
|
||||
#check if there are enough arguments, if not, exit with an error
|
||||
|
||||
|
|
@ -17,21 +17,98 @@ nonversionedpaths=( $4 )
|
|||
checkouts=( $5 )
|
||||
|
||||
#echo svnbin: $svnbin
|
||||
#echo subversionedpaths: $subversionedpaths
|
||||
#echo subversionedpathsCommitBools: $subversionedpathsCommitBools
|
||||
#echo nonversionedpaths: $nonversionedpaths
|
||||
#echo checkouts: $checkouts
|
||||
#echo subversionedpaths: ${subversionedpaths[@]}
|
||||
#echo subversionedpathsCommitBools: ${subversionedpathsCommitBools[@]}
|
||||
#echo nonversionedpaths: ${nonversionedpaths[@]}
|
||||
#echo checkouts: ${checkouts[@]}
|
||||
|
||||
|
||||
|
||||
#TODO
|
||||
#
|
||||
# silence "is already under version control" messages
|
||||
# after a "revert $x" you can silence the "skipped '$x'"
|
||||
#
|
||||
#TODO
|
||||
#
|
||||
#
|
||||
function subversionSingleStateDir {
|
||||
|
||||
excludelist=( "." ".." ".svn" );
|
||||
|
||||
cd $1;
|
||||
echo cd $1;
|
||||
|
||||
empty=$(ls)
|
||||
|
||||
if [ "$empty" = "" ] ; then
|
||||
allsubitems=(); #no subfiles / dirs
|
||||
else
|
||||
allsubitems=( $(echo *) $(echo .*) ) #there are subfiles / dirs,also adds hidden items
|
||||
fi
|
||||
|
||||
for subitem in ${allsubitems[@]}
|
||||
do
|
||||
if [ "$subitem" = ".svn" ]; then
|
||||
allsubitems=( $(svn -N stat | sed -n '/^?/p' | sed 's/? //' | tr -d "\12") ) #there are subfiles, and theyre already versioned
|
||||
fi
|
||||
done
|
||||
|
||||
#echo "Allsubitems ${allsubitems[@]}"
|
||||
|
||||
subitems=();
|
||||
for subitem in ${allsubitems[@]} #add all, were going to exlucde explicity stated direct versioned-subdirs or explicity stated nonversioned-subdirs
|
||||
do #this is only to prevent some warnings, ultimately we would like svn add to have a option 'exclude dirs'
|
||||
|
||||
exclude=0;
|
||||
|
||||
for excl in ${excludelist[@]} #check if the subitem is in the list of excluded dirs
|
||||
do
|
||||
if [ "$excl" = "$subitem" ]; then
|
||||
exclude=1;
|
||||
#echo "exclude $excl"
|
||||
fi
|
||||
done
|
||||
|
||||
subitem="$(pwd)/$subitem"; #create the full path
|
||||
|
||||
if test -d $subitem; then #the subitem (file or a dir) may be a dir, so we add a / to the end
|
||||
subitem="$subitem/";
|
||||
fi
|
||||
|
||||
for svnp in ${subversionedpaths[@]} #check if the subitem is in the list of subverioned paths
|
||||
do
|
||||
if [ "$svnp" = "$subitem" ]; then
|
||||
exclude=1;
|
||||
#echo "exclude versioned $svnp"
|
||||
fi
|
||||
done
|
||||
|
||||
for nonvp in ${nonversionedpaths[@]} #check if the subitem is in the list of dirs that aren't supposed to be versioned
|
||||
do
|
||||
if [ "$nonvp" = "$subitem" ]; then
|
||||
exclude=1;
|
||||
#echo "exclude nonversioned $svnp"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $exclude = 0 ]; then #Exclude the subitem if nessecary
|
||||
subitems[${#subitems[*]}]=$subitem
|
||||
fi
|
||||
done
|
||||
|
||||
#echo ${subitems[@]}
|
||||
|
||||
for item in ${subitems[@]}
|
||||
do
|
||||
if test -d $item; then #add or go recursive subitems
|
||||
$debug svn -N add $item #NON recursively add the dir
|
||||
subversionSingleStateDir $item
|
||||
else
|
||||
$debug svn add $item
|
||||
fi
|
||||
done
|
||||
|
||||
}
|
||||
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
|
||||
i=0
|
||||
for path in ${subversionedpaths[@]}
|
||||
|
|
@ -40,7 +117,7 @@ do
|
|||
cd $path;
|
||||
|
||||
output=$($svnbin stat 2>&1 | grep "is not a working copy");
|
||||
if [ "$output" != "" ] ; then #if the dir exists but is not yet an svn dir: create repos, if it doenst exits (is removed or something) than we dont do anything
|
||||
if [ "$output" != "" ] ; then #if the dir exists but is not yet an svn dir: checkout repos, if it doenst exits (is removed or something) than we dont do anything
|
||||
$debug ${checkouts[$i]};
|
||||
fi
|
||||
|
||||
|
|
@ -48,57 +125,10 @@ do
|
|||
|
||||
echo "Entering $path"
|
||||
|
||||
allsubitems=( $(echo *) ) #TODO, maybe also add hidden files starting with a '.' , but we dont want to add .svn dirs
|
||||
#TODO2 maybe do something with svn stat to speed up ?
|
||||
subitems=();
|
||||
for subitem in ${allsubitems[@]} #add all, were going to exlucde explicity stated direct versioned-subdirs or explicity stated nonversioned-subdirs
|
||||
do #this is only to prevent some warnings, ultimately we would like svn add to have a option 'exclude dirs'
|
||||
subitem="$(pwd)/$subitem";
|
||||
|
||||
if test -d $subitem; then #the subitem (file or a dir) may be a dir, so we add a / to the end
|
||||
subitem="$subitem/";
|
||||
fi
|
||||
|
||||
exclude=0;
|
||||
|
||||
for svnp in ${subversionedpaths[@]} #check if the subitem is in the list of subverioned paths
|
||||
do
|
||||
if [ "$svnp" = "$subitem" ]; then
|
||||
exclude=1;
|
||||
#echo "exclude versioned $svnp"
|
||||
fi
|
||||
done
|
||||
|
||||
for nonvp in ${nonversionedpaths[@]} #check if the subitem is in the list of dirs that aren't supposed to be versioned
|
||||
do
|
||||
if [ "$nonvp" = "$subitem" ]; then
|
||||
exclude=1;
|
||||
#echo "exclude nonversioned $svnp"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $exclude = 0 ]; then #Exclude the subitem if nessecary
|
||||
subitems[${#subitems[*]}]=$subitem
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$subitems" != "" ]; then
|
||||
echo "adding ${subitems[@]}"
|
||||
$debug svn add ${subitems[@]} #add all subitems
|
||||
|
||||
for revpath in ${nonversionedpaths[@]} #We need to revert sub-sub* dirs, since these havent been excluded
|
||||
do
|
||||
if test -d $revpath; then
|
||||
if [ "${revpath:0:${#path}}" == "$path" ]; then
|
||||
echo "Revert $revpath";
|
||||
$debug svn revert $revpath;
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
$debug svn -m "" commit; #Finally, we commit
|
||||
fi
|
||||
subversionSingleStateDir $path;
|
||||
|
||||
cd $path #now that everything is added we can commit
|
||||
$debug svn -m "" commit;
|
||||
fi
|
||||
|
||||
cd - &> /dev/null;
|
||||
|
|
@ -106,3 +136,7 @@ do
|
|||
let "i+=1"
|
||||
done
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -470,7 +470,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
|
|||
s = s + "/";
|
||||
|
||||
//Remove the / at the beginning if it's there
|
||||
if(s[0] == '/')
|
||||
if(s[0] == '/' && s.length() != 1)
|
||||
s = s.substr(1, s.length());
|
||||
|
||||
if(s == stateRootRepos)
|
||||
|
|
@ -619,14 +619,18 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
|
|||
|
||||
printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'") % drvName % drvPath);
|
||||
|
||||
/* TODO Write updated (no need to rebuild) state derivations to a special table, so they can be updated at build time */
|
||||
/* Write updated (no need to rebuild) state derivations to the database, so they can be updated at build time */
|
||||
if(enableState && !disableState){
|
||||
Path deriver = queryDeriver(noTxn, outPath); //query deriver
|
||||
if(deriver != drvPath){
|
||||
store->setUpdatedStateDerivation(drvPath, outPath);
|
||||
if(store->isValidPath(outPath)){ //Only add when the path is already valid
|
||||
Path deriver = queryDeriver(noTxn, outPath); //query the deriver
|
||||
if(deriver != drvPath){
|
||||
store->addUpdatedStateDerivation(drvPath, outPath);
|
||||
}
|
||||
}
|
||||
//TODO Also add when path is not already valid, which drv does it take at build time, the latest ... guess so .. so we dont need to add?
|
||||
}
|
||||
|
||||
|
||||
/* Optimisation, but required in read-only mode! because in that
|
||||
case we don't actually write store expressions, so we can't
|
||||
read them later. */
|
||||
|
|
|
|||
|
|
@ -2477,12 +2477,14 @@ void LocalStore::buildDerivations(const PathSet & drvPaths)
|
|||
startNest(nest, lvlDebug,
|
||||
format("building %1%") % showPaths(drvPaths));
|
||||
|
||||
//Just before we build, we resolve the multiple derivations linked to one store path issue, by choosing the latest derivation
|
||||
store->updateAllStateDerivations();
|
||||
|
||||
Worker worker;
|
||||
|
||||
Goals goals;
|
||||
for (PathSet::const_iterator i = drvPaths.begin(); i != drvPaths.end(); ++i){
|
||||
goals.insert(worker.makeDerivationGoal(*i));
|
||||
printMsg(lvlError, format("No component build, but state check: %1%") % *i);
|
||||
}
|
||||
|
||||
worker.run(goals);
|
||||
|
|
|
|||
|
|
@ -1223,41 +1223,82 @@ PathSet LocalStore::getStateReferencesClosure(const Path & path)
|
|||
}
|
||||
|
||||
|
||||
//TODO
|
||||
void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
|
||||
void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
|
||||
{
|
||||
Transaction txn(nixDB);
|
||||
|
||||
//Check wheter were not duplicating an entry, if so, than we wont have to do anything
|
||||
//TODO
|
||||
if(false){
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
Strings data;
|
||||
data.push_back(storepath);
|
||||
|
||||
time_t timestamp;
|
||||
time (×tamp);
|
||||
string timestamp_s = time_t2string(timestamp);
|
||||
printMsg(lvlError, format("Adding new drv (%1%) to replace drv of old component (%2%) with timestamp: %3%") % newdrv % storepath % timestamp_s);
|
||||
data.push_back(timestamp_s); //create a timestamp to remember which one was last inserted
|
||||
|
||||
nixDB.setStrings(txn, dbUpdatedDerivations, newdrv, data);
|
||||
|
||||
//TODO check wheter were not duplicating an entry !!
|
||||
|
||||
txn.commit();
|
||||
}
|
||||
|
||||
//TODO
|
||||
void LocalStore::setUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
|
||||
void LocalStore::addUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
|
||||
{
|
||||
nix::setUpdatedStateDerivation(newdrv, storepath);
|
||||
nix::addUpdatedStateDerivation(newdrv, storepath);
|
||||
}
|
||||
|
||||
//TODO
|
||||
Path updateStateDerivation(const Path & storepath)
|
||||
void updateAllStateDerivations()
|
||||
{
|
||||
Transaction txn(nixDB);
|
||||
|
||||
Path drvPath = queryDeriver(txn, storepath);
|
||||
printMsg(lvlError, format("Current DRV: %1%") % drvPath);
|
||||
Strings unique_paths;
|
||||
|
||||
Strings keys;
|
||||
nixDB.enumTable(txn, dbUpdatedDerivations, keys);
|
||||
for (Strings::iterator i = keys.begin(); i != keys.end(); ++i)
|
||||
{
|
||||
string drv_key = *i; //the key is the derivation
|
||||
Strings data;
|
||||
nixDB.queryStrings(txn, dbUpdatedDerivations, drv_key, data);
|
||||
string path = data.front();
|
||||
bool exists = false;
|
||||
|
||||
for (Strings::iterator j = unique_paths.begin(); j != unique_paths.end(); ++j)
|
||||
{
|
||||
string unique_path = *j;
|
||||
|
||||
if(path == unique_path)
|
||||
exists = true;
|
||||
}
|
||||
|
||||
if(!exists)
|
||||
unique_paths.push_back(path);
|
||||
}
|
||||
|
||||
for (Strings::iterator i = unique_paths.begin(); i != unique_paths.end(); ++i)
|
||||
{
|
||||
string path = *i;
|
||||
printMsg(lvlError, format("Unique: %1%") % path);
|
||||
store->updateStateDerivation(txn, path); //TODO replace store->
|
||||
}
|
||||
|
||||
txn.commit();
|
||||
}
|
||||
|
||||
void LocalStore::updateAllStateDerivations()
|
||||
{
|
||||
nix::updateAllStateDerivations();
|
||||
}
|
||||
|
||||
|
||||
void updateStateDerivation(const Transaction & txn, const Path & storepath)
|
||||
{
|
||||
Path drvPath = queryDeriver(txn, storepath);
|
||||
Path originalDerivation = drvPath;
|
||||
Path newDerivation = drvPath; //the new drv path first equals the old one until a new one is found
|
||||
int timestamp = 0;
|
||||
|
||||
|
|
@ -1280,15 +1321,12 @@ Path updateStateDerivation(const Path & storepath)
|
|||
string2Int(gettimestamp, gettimestamp_i);
|
||||
|
||||
if(gettimestamp_i == timestamp)
|
||||
throw Error(format("Error! Multiple changes at the same time of derivation: `%1%' with timestamp") % drv_key); //TODO delete the mulitple changes ??
|
||||
throw Error(format("Error! Multiple changes at store path %4% at the same time: derivations: `%1%' and `%2%' with timestamp `%3%'") % newDerivation % drv_key % timestamp % storepath);
|
||||
|
||||
if(timestamp == 0 || gettimestamp_i > timestamp){ //we choose the new derivation as the latest submitted derivation
|
||||
printMsg(lvlError, format("Replacing old drv (%1%) with new drv (%2%) with timestamp: %3%") % newDerivation % drv_key % gettimestamp_i);
|
||||
//printMsg(lvlError, format("Replacing at store path %4% the old drv (%1%) with new drv (%2%) with timestamp: %3%") % newDerivation % drv_key % gettimestamp_i % storepath);
|
||||
newDerivation = drv_key;
|
||||
timestamp = gettimestamp_i;
|
||||
|
||||
//Replace the old deriver link in the derivers database (TODO, and delete old deriver path????????)
|
||||
setDeriver(txn, storepath, newDerivation);
|
||||
}
|
||||
|
||||
//Always Remove the old updatelink in the dbUpdatedDerivations
|
||||
|
|
@ -1296,14 +1334,21 @@ Path updateStateDerivation(const Path & storepath)
|
|||
}
|
||||
}
|
||||
|
||||
txn.commit();
|
||||
return newDerivation;
|
||||
if(originalDerivation != newDerivation) //only update if neccecary
|
||||
{
|
||||
//Replace the old deriver link in the derivers database (TODO, maybe delete old deriver path????????)
|
||||
setDeriver(txn, storepath, newDerivation);
|
||||
|
||||
//Call the stateUpdate function for the new derivation? Yes since this function is called at build time
|
||||
printMsg(lvlError, format("Calling new state drv %1% for storepath %2%") % newDerivation % storepath);
|
||||
//TODO check wheter we update before or after ??
|
||||
//TODO
|
||||
}
|
||||
}
|
||||
|
||||
//TODO
|
||||
Path LocalStore::updateStateDerivation(const Path & storepath)
|
||||
void LocalStore::updateStateDerivation(const Transaction & txn, const Path & storepath)
|
||||
{
|
||||
return nix::updateStateDerivation(storepath);
|
||||
nix::updateStateDerivation(txn, storepath);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -84,9 +84,11 @@ public:
|
|||
|
||||
PathSet getStateReferencesClosure(const Path & path);
|
||||
|
||||
void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath);
|
||||
void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath);
|
||||
|
||||
Path updateStateDerivation(const Path & storepath);
|
||||
void updateStateDerivation(const Transaction & txn, const Path & storepath);
|
||||
|
||||
void updateAllStateDerivations();
|
||||
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -403,16 +403,21 @@ PathSet RemoteStore::getStateReferencesClosure(const Path & path)
|
|||
}
|
||||
|
||||
//TODO
|
||||
void RemoteStore::setUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
|
||||
void RemoteStore::addUpdatedStateDerivation(const Path & newdrv, const Path & storepath)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
//TODO
|
||||
Path RemoteStore::updateStateDerivation(const Path & storepath)
|
||||
void RemoteStore::updateStateDerivation(const Transaction & txn, const Path & storepath)
|
||||
{
|
||||
Path p;
|
||||
return p;
|
||||
|
||||
}
|
||||
|
||||
//TODO
|
||||
void RemoteStore::updateAllStateDerivations()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -72,9 +72,11 @@ public:
|
|||
|
||||
PathSet getStateReferencesClosure(const Path & path);
|
||||
|
||||
void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath);
|
||||
void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath);
|
||||
|
||||
Path updateStateDerivation(const Path & storepath);
|
||||
void updateStateDerivation(const Transaction & txn, const Path & storepath);
|
||||
|
||||
void updateAllStateDerivations();
|
||||
|
||||
|
||||
private:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
#include "hash.hh"
|
||||
#include "serialise.hh"
|
||||
#include "derivations.hh"
|
||||
#include "db.hh"
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
|
@ -194,10 +195,13 @@ public:
|
|||
virtual PathSet getStateReferencesClosure(const Path & path) = 0;
|
||||
|
||||
/* TODO */
|
||||
virtual void setUpdatedStateDerivation(const Path & newdrv, const Path & storepath) = 0;
|
||||
virtual void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath) = 0;
|
||||
|
||||
/* TODO */
|
||||
virtual Path updateStateDerivation(const Path & storepath) = 0;
|
||||
virtual void updateStateDerivation(const Transaction & txn, const Path & storepath) = 0;
|
||||
|
||||
/* TODO */
|
||||
virtual void updateAllStateDerivations() = 0;
|
||||
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -16,9 +16,12 @@
|
|||
namespace nix {
|
||||
|
||||
|
||||
void dsfsdfas()
|
||||
void updatedStateDerivation(Path storePath)
|
||||
{
|
||||
//Remove the old .svn folders
|
||||
|
||||
//Create new repositorys, or use existing...
|
||||
//createStateDirs already does that ...
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1016,7 +1016,7 @@ string trim(const string & s) {
|
|||
|
||||
void executeAndPrintShellCommand(const string & command, const string & commandName)
|
||||
{
|
||||
string tempoutput = "svnoutput.txt";
|
||||
string tempoutput = "/tmp/svnoutput.txt";
|
||||
string newcommand = command + " &> " + tempoutput; //the &> sends also stderr to stdout
|
||||
|
||||
int kidstatus, deadpid;
|
||||
|
|
|
|||
|
|
@ -102,7 +102,9 @@ static void opRunComponent(Strings opFlags, Strings opArgs)
|
|||
|
||||
//Check if component is a state component !!!
|
||||
|
||||
//Wait for locks?
|
||||
//Check for locks ...
|
||||
//add locks ... ?
|
||||
//svn lock ... ?
|
||||
|
||||
//******************* Run the component
|
||||
//TODO
|
||||
|
|
@ -156,7 +158,10 @@ static void opRunComponent(Strings opFlags, Strings opArgs)
|
|||
DerivationStateOutputDir d = i->second;
|
||||
|
||||
string thisdir = d.path;
|
||||
|
||||
string fullstatedir = statePath + "/" + thisdir;
|
||||
if(thisdir == "/") //exception for the root dir
|
||||
fullstatedir = statePath + "/";
|
||||
Path statePath = fullstatedir; //TODO call coerce function
|
||||
|
||||
if(d.type == "none"){
|
||||
|
|
@ -229,11 +234,13 @@ void run(Strings args)
|
|||
Strings opFlags, opArgs;
|
||||
Operation op = 0;
|
||||
|
||||
/* test */
|
||||
/* test *
|
||||
store = openStore();
|
||||
Path p = "/nix/store/l569q3a2cfx834mcf3vhwczjgbaljnp7-hellohardcodedstateworld-1.0";
|
||||
store->setUpdatedStateDerivation("/nix/store/63xcbrk3v5nbn9qla7rwnx6rvz3iqm5l-hellohardcodedstateworld-1.0.drv", p);
|
||||
store->updateStateDerivation(p);
|
||||
Path p = "/nix/store/l569q3a2cfx834mcf3vhwczjgbaljnp7-hellohardcodedstateworld-1.0"; //
|
||||
store->addUpdatedStateDerivation("/nix/store/63xcbrk3v5nbn9qla7rwnx6rvz3iqm5l-hellohardcodedstateworld-1.0.drv", p); //
|
||||
Path p2 = "/nix/store/4ycq45hsgc8yaj4vwafx3lgd473jaqwg-hellohardcodedstateworld-1.0";
|
||||
store->addUpdatedStateDerivation("/nix/store/s6wggk924jx0gcb0l29ra4g9fxa3b4pp-hellohardcodedstateworld-1.0.drv", p2); //
|
||||
store->updateAllStateDerivations();
|
||||
return;
|
||||
/* test */
|
||||
|
||||
|
|
@ -261,6 +268,14 @@ void run(Strings args)
|
|||
|
||||
TODO update getDerivation in nix-store to handle state indentifiers
|
||||
|
||||
--update state drv
|
||||
|
||||
--revert-to-state (recursive revert...)
|
||||
|
||||
--delete state?
|
||||
|
||||
--
|
||||
|
||||
*/
|
||||
|
||||
else
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue