Merged to R9439. Fixed a computeFSClosure bug. The state garbage colletor basically works, Missing items: State locks, shared state and Topological sort

install_full.sh

@@ -1,5 +1,7 @@
 #! /bin/sh -e
 
+make clean
+
 export nixstatepath=/nixstate2/nix
 export ACLOCAL_PATH=/home/wouterdb/.nix-profile/share/aclocal
 

scripts/optimise-store.pl

@@ -1,91 +0,0 @@
-#! /usr/bin/perl -w
-
-use strict;
-use File::Basename;
-
-
-my @paths = ("/nix/store");
-
-
-print "hashing...\n";
-
-my $hashList = "/tmp/nix-optimise-hash-list";
-
-system("find @paths -type f -print0 | xargs -0 md5sum -- > $hashList") == 0
-    or die "cannot hash store files";
-
-
-print "sorting by hash...\n";
-
-system("sort $hashList > $hashList.sorted") == 0
-    or die "cannot sort list";
-
-
-sub atomicLink {
-    my $target = shift;
-    my $new = shift;
-    my $tmpNew = "${new}_optimise.$$";
-
-    # Make the directory writable temporarily.
-    my $dir = dirname $new;
-    my @st = stat $dir or die;
-
-    chmod ($st[2] | 0200, $dir) or die "cannot make `$dir' writable: $!";
-    
-    link $target, $tmpNew or die "cannot create hard link `$tmpNew': $!";
-
-    rename $tmpNew, $new or die "cannot rename `$tmpNew' to `$new': $!";
-
-    chmod ($st[2], $dir) or die "cannot restore permission on `$dir': $!";
-    utime ($st[8], $st[9], $dir) or die "cannot restore timestamp on `$dir': $!";
-}
-
-
-print "hard-linking...\n";
-
-open LIST, "<$hashList.sorted" or die;
-
-my $prevFile;
-my $prevHash;
-my $prevInode;
-my $prevExec;
-
-my $totalSpace = 0;
-my $savedSpace = 0;
-
-while (<LIST>) {
-    /^([0-9a-f]*)\s+(.*)$/ or die;
-    my $curFile = $2;
-    my $curHash = $1;
-
-    my @st = stat $curFile or die;
-    next if ($st[2] & 0222) != 0; # skip writable files
-
-    my $fileSize = $st[7];
-    $totalSpace += $fileSize;
-    my $isExec = ($st[2] & 0111) == 0111;
-
-    if (defined $prevHash && $curHash eq $prevHash
-        && $prevExec == $isExec)
-    {
-        
-        if ($st[1] != $prevInode) {
-            print "$curFile = $prevFile\n";
-            atomicLink $prevFile, $curFile;
-            $savedSpace += $fileSize;
-        }
-        
-    } else {
-        $prevFile = $curFile;
-        $prevHash = $curHash;
-        $prevInode = $st[1];
-        $prevExec = ($st[2] & 0111) == 0111;
-    }
-}
-
-print "total space = $totalSpace\n";
-print "saved space = $savedSpace\n";
-my $savings = ($savedSpace / $totalSpace) * 100.0;
-print "savings = $savings %\n";
-
-close LIST;

src/libstore/gc.cc

@@ -452,11 +452,11 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
     /* Acquire the global GC root.  This prevents
        a) New roots from being added.
        b) Processes from creating new temporary root files. */
-    AutoCloseFD fdGCLock = openGCLock(ltWrite);
+    AutoCloseFD fdGCLock = openGCLock(ltWrite);												//TODO !!!!!!!!!!!!!!!!!!! GET STATE LOCKS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
-    Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true);					//TODO Also find state roots --> nah ? TODO include sharing of state.
+    Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true);
 
     PathSet roots;
     for (Roots::iterator i = rootMap.begin(); i != rootMap.end(); ++i)
@@ -480,9 +480,10 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
     PathSet livePaths;
     for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i){
     	printMsg(lvlError, format("CHECK '%1%'") % *i);
-        computeFSClosure(canonPath(*i), livePaths, true, false, 0);							//TODO !!!!!!!!!! ALSO STATEPATHS TRUE???
+        computeFSClosure(canonPath(*i), livePaths, true, true, 0);
     }
 
+	printMsg(lvlError, format("STAGE X"));
 
     if (gcKeepDerivations) {
         for (PathSet::iterator i = livePaths.begin();
@@ -496,28 +497,29 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
         		for (PathSet::const_iterator j = derivers.begin(); j != derivers.end(); ++j)
 					// We send each drv to computeFSClosure
         			if (*j != "" && store->isValidPath(*j))
-                		computeFSClosure(*j, livePaths, true, false, 0);					//TODO !!!!!!!!!! ALSO STATEPATHS TRUE???
+                		computeFSClosure(*j, livePaths, true, true, 0);
         	}
 			else if (store->isValidPath(*i)){
 				printMsg(lvlError, format("CHECK-STORE '%1%'") % *i);
             	Path deriver = store->queryDeriver(*i);
-            	printMsg(lvlError, format("CHECK-STORE DRV '%1%'") % deriver);
+            	//printMsg(lvlError, format("CHECK-STORE DRV '%1%'") % deriver);
 				
                /* Note that the deriver need not be valid (e.g., if we
                previously ran the collector with `gcKeepDerivations'
                turned off). */
 				if (deriver != "" && store->isValidPath(deriver))
-                	computeFSClosure(deriver, livePaths, true, false, 0);					//TODO !!!!!!!!!! ALSO STATEPATHS TRUE???
+                	computeFSClosure(deriver, livePaths, true, true, 0);
             }
+			else{
+				printMsg(lvlError, format("CHECK-STATE '%1%'") % *i);
 				
-			//else if (store->isValidStatePath(*i)){
+				//TODO also get its deriver???? for if its component is gone !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 				
-			else 
+			}    
-				throw Error(format("path `%1%' is not a valid state/store path") % *i);    
         }
     }
     
-    printMsg(lvlError, format("STAGE X"));
+    printMsg(lvlError, format("STAGE Y"));
 
     if (gcKeepOutputs) {
         /* Hmz, identical to storePathRequisites in nix-store. */
@@ -529,7 +531,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
                 for (DerivationOutputs::iterator j = drv.outputs.begin();
                      j != drv.outputs.end(); ++j)
                     if (store->isValidPath(j->second.path))
-                        computeFSClosure(j->second.path, livePaths, true, false, 0);					//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WE (MAY) ALSO NEED TO KEEP STATE
+                        computeFSClosure(j->second.path, livePaths, true, true, 0);					//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WHAT ARE (STATE) OUTPUTS ????????????
             }
     }
 
@@ -567,24 +569,37 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
        that is not currently in in `livePaths' or `tempRootsClosed'
        can be deleted. */
     
-    /* Read the Nix store directory to find all currently existing
+    ///TODO Calc get shared paths																	//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    //for: livePaths, tempRootsClosed
+    PathSet sharedWith_livePaths_tempRootsClosed;
+    
+    /* Read the Nix store and state directory's to find all currently existing
        paths. */
     PathSet storePathSet;
+    PathSet statePathSet;
+    
     if (action != gcDeleteSpecific) {
+    	
         Paths entries = readDirectory(nixStore);
+        Paths stateEntries = readDirectory(nixStoreState);
+        
         for (Paths::iterator i = entries.begin(); i != entries.end(); ++i)
             storePathSet.insert(canonPath(nixStore + "/" + *i));
+        
+        for (Paths::iterator i = stateEntries.begin(); i != stateEntries.end(); ++i)
+            statePathSet.insert(canonPath(nixStoreState + "/" + *i));
+        
     } else {
         for (PathSet::iterator i = pathsToDelete.begin();
              i != pathsToDelete.end(); ++i)
         {
-            assertStorePath(*i);
+            assertStorePath(*i);											//TODO ASSERTSTATEPATH !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
             storePathSet.insert(*i);
         }
     }
 
     /* Topologically sort them under the `referrers' relation.  That
-       is, a < b iff a is in referrers(b).  This gives us the order in
+       is, a < b if a is in referrers(b).  This gives us the order in
        which things can be deleted safely. */
     /* !!! when we have multiple output paths per derivation, this
        will not work anymore because we get cycles. */
@@ -654,6 +669,95 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
 #endif
         }
     }
+
+    /* Try to delete state paths. */
+    /* Topologically sort them under the `referrers' relation.  That
+       is, a < b if a is in referrers(b).  This gives us the order in
+       which things can be deleted safely. */
+    /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! when we have multiple output paths per derivation, this
+       will not work anymore because we get cycles. */
+    Paths statePaths;
+    for (PathSet::iterator i = statePathSet.begin(); i != statePathSet.end(); ++i)
+    	statePaths.push_back(*i);
+    
+    
+    //
+    
+    for (Paths::iterator i = statePaths.begin(); i != statePaths.end(); ++i) {
+    	
+    	debug(format("considering deletion of state path `%1%'") % *i);
+        
+        if (livePaths.find(*i) != livePaths.end()) {
+            if (action == gcDeleteSpecific)
+                throw Error(format("cannot delete state path `%1%' since it is still alive") % *i);
+            debug(format("live state path `%1%'") % *i);
+            continue;
+        }
+
+        if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
+            debug(format("temporary root `%1%'") % *i);
+            continue;
+        }
+        
+        if (sharedWith_livePaths_tempRootsClosed.find(*i) != sharedWith_livePaths_tempRootsClosed.end()) {
+            debug(format("State path `%1%' is shared with another live path") % *i);
+            continue;
+        }
+        
+        debug(format("dead path `%1%'") % *i);
+        result.insert(*i);
+
+        /* If just returning the set of dead paths, we also return the
+           space that would be freed if we deleted them. */
+        if (action == gcReturnDead)
+            bytesFreed += computePathSize(*i);
+
+        if (action == gcDeleteDead || action == gcDeleteSpecific) {
+
+//TODO !!!!!!!!!!!!!!!!!!!!!! state locks
+/*
+#ifndef __CYGWIN__
+            AutoCloseFD fdLock;
+
+            /* Only delete a lock file if we can acquire a write lock
+               on it.  That means that it's either stale, or the
+               process that created it hasn't locked it yet.  In the
+               latter case the other process will detect that we
+               deleted the lock, and retry (see pathlocks.cc). * /
+            if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
+                fdLock = openLockFile(*i, false);
+                if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) {
+                    debug(format("skipping active lock `%1%'") % *i);
+                    continue;
+                }
+            }
+#endif
+*/
+
+            if (!pathExists(*i)) continue;
+                
+            printMsg(lvlInfo, format("deleting state path `%1%'") % *i);
+            
+            /* Okay, it's safe to delete. */
+            try {
+                unsigned long long freed;
+                //deleteFromState(*i, freed);											//TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! PUT BACK ON
+                bytesFreed += freed;
+            } catch (PathInUse & e) {
+                printMsg(lvlError, format("warning: %1%") % e.msg());
+            }
+
+/*
+#ifndef __CYGWIN__
+            if (fdLock != -1)
+                /* Write token to stale (deleted) lock file. * /
+                writeFull(fdLock, (const unsigned char *) "d", 1);
+#endif
+*/
+
+        }
+        
+    }
 }
 
  

src/libstore/local-store.cc

@@ -288,7 +288,7 @@ void copyPath(const Path & src, const Path & dst, PathFilter & filter)
 }
 
 
-static void _canonicalisePathMetaData(const Path & path)
+static void _canonicalisePathMetaData(const Path & path, bool recurse)
 {
     checkInterrupt();
 
@@ -337,17 +337,17 @@ static void _canonicalisePathMetaData(const Path & path)
 
     }
 
-    if (S_ISDIR(st.st_mode)) {
+    if (recurse && S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
 	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
-	    _canonicalisePathMetaData(path + "/" + *i);
+	    _canonicalisePathMetaData(path + "/" + *i, true);
     }
 }
 
 
 void canonicalisePathMetaData(const Path & path)
 {
-    _canonicalisePathMetaData(path);
+    _canonicalisePathMetaData(path, true);
 
     /* On platforms that don't have lchown(), the top-level path can't
        be a symlink, since we can't change its ownership. */
@@ -1118,7 +1118,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
        consistent metadata. */
     Transaction txn(nixDB);
     addTempRoot(path);
-    if (!isValidPath(path))
+    if (!isValidPathTxn(txn, path))
         throw Error(format("path `%1%' is not valid") % path);
 
     HashAndWriteSink hashAndWriteSink(sink);
@@ -1418,7 +1418,7 @@ void verifyStore(bool checkContents)
         }
         else {
             PathSet references;
-            queryXReferencesTxn(txn, *i, references, true, -1);				//TODO
+            queryXReferencesTxn(txn, *i, references, true, 0);				//TODO
             for (PathSet::iterator j = references.begin();
                  j != references.end(); ++j)
             {
@@ -1843,6 +1843,125 @@ void LocalStore::revertToRevision(const Path & statePath, const unsigned int rev
 }
 
 
+typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath;
+
+
+static void makeWritable(const Path & path)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+	throw SysError(format("getting attributes of path `%1%'") % path);
+    if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
+        throw SysError(format("changing writability of `%1%'") % path);
+}
+
+
+static void hashAndLink(bool dryRun, HashToPath & hashToPath,
+    OptimiseStats & stats, const Path & path)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+	throw SysError(format("getting attributes of path `%1%'") % path);
+
+    /* Sometimes SNAFUs can cause files in the Nix store to be
+       modified, in particular when running programs as root under
+       NixOS (example: $fontconfig/var/cache being modified).  Skip
+       those files. */
+    if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
+        printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
+        return;
+    }
+
+    /* We can hard link regular files and symlinks. */
+    if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) {
+
+        /* Hash the file.  Note that hashPath() returns the hash over
+           the NAR serialisation, which includes the execute bit on
+           the file.  Thus, executable and non-executable files with
+           the same contents *won't* be linked (which is good because
+           otherwise the permissions would be screwed up).
+
+           Also note that if `path' is a symlink, then we're hashing
+           the contents of the symlink (i.e. the result of
+           readlink()), not the contents of the target (which may not
+           even exist). */
+        Hash hash = hashPath(htSHA256, path);
+        stats.totalFiles++;
+        printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
+
+        std::pair<Path, ino_t> prevPath = hashToPath[hash];
+        
+        if (prevPath.first == "") {
+            hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
+            return;
+        }
+            
+        /* Yes!  We've seen a file with the same contents.  Replace
+           the current file with a hard link to that file. */
+        stats.sameContents++;
+        if (prevPath.second == st.st_ino) {
+            printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first);
+            return;
+        }
+        
+        if (!dryRun) {
+            
+            printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first);
+
+            Path tempLink = (format("%1%.tmp-%2%-%3%")
+                % path % getpid() % rand()).str();
+
+            /* Make the containing directory writable, but only if
+               it's not the store itself (we don't want or need to
+               mess with  its permissions). */
+            bool mustToggle = !isStorePath(path);
+            if (mustToggle) makeWritable(dirOf(path));
+        
+            if (link(prevPath.first.c_str(), tempLink.c_str()) == -1)
+                throw SysError(format("cannot link `%1%' to `%2%'")
+                    % tempLink % prevPath.first);
+
+            /* Atomically replace the old file with the new hard link. */
+            if (rename(tempLink.c_str(), path.c_str()) == -1)
+                throw SysError(format("cannot rename `%1%' to `%2%'")
+                    % tempLink % path);
+
+            /* Make the directory read-only again and reset its
+               timestamp back to 0. */
+            if (mustToggle) _canonicalisePathMetaData(dirOf(path), false);
+            
+        } else
+            printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
+        
+        stats.filesLinked++;
+        stats.bytesFreed += st.st_size;
+    }
+
+    if (S_ISDIR(st.st_mode)) {
+        Strings names = readDirectory(path);
+	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+	    hashAndLink(dryRun, hashToPath, stats, path + "/" + *i);
+    }
+}
+
+
+void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
+{
+    HashToPath hashToPath;
+    
+    Paths paths;
+    PathSet validPaths;
+    nixDB.enumTable(noTxn, dbValidPaths, paths);
+
+    for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
+        addTempRoot(*i);
+        if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
+        startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
+        hashAndLink(dryRun, hashToPath, stats, *i);
+    }
+}
+
+
 /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */
 static void upgradeStore07()
 {
@@ -1912,7 +2031,7 @@ static void upgradeStore07()
                 }
 
                 PathSet prevReferences;
-                queryXReferencesTxn(txn, path, prevReferences, true, -1);
+                queryXReferencesTxn(txn, path, prevReferences, true, 0);
                 if (prevReferences.size() > 0 && references != prevReferences)
                     printMsg(lvlError, format("warning: conflicting references for `%1%'") % path);
 

src/libstore/local-store.hh

@@ -21,6 +21,20 @@ const int nixSchemaVersion = 4;
 extern string drvsLogDir;
 
 
+struct OptimiseStats
+{
+    unsigned long totalFiles;
+    unsigned long sameContents;
+    unsigned long filesLinked;
+    unsigned long long bytesFreed;
+    OptimiseStats()
+    {
+        totalFiles = sameContents = filesLinked = 0;
+        bytesFreed = 0;
+    }
+};
+
+
 class LocalStore : public StoreAPI
 {
 private:
@@ -125,6 +139,10 @@ public:
 	void shareState(const Path & from, const Path & to, const bool snapshot);
 	
 	void unShareState(const Path & path, const bool branch, const bool restoreOld);
+
+    /* Optimise the disk space usage of the Nix store by hard-linking
+       files with the same contents. */
+    void optimiseStore(bool dryRun, OptimiseStats & stats);
 };
 
 

src/libstore/misc.cc

@@ -37,6 +37,10 @@ void computeFSClosure(const Path & path, PathSet & paths, const bool & withCompo
 	computeFSClosureTxn(noTxn, path, paths, withComponents, withState, revision, flipDirection);
 }
 
+/*
+ * Notice that the incoming 'paths' parameter can already be filled and only needs to be expanded
+ * Notice that we CANNOT change the order of the existing paths
+ */
 void computeFSClosureTxn(const Transaction & txn, const Path & path, PathSet & paths, const bool & withComponents, const bool & withState, const int revision, bool flipDirection)
 {
 	PathSet allPaths;
@@ -45,27 +49,23 @@ void computeFSClosureTxn(const Transaction & txn, const Path & path, PathSet & p
 	if(!withComponents && !withState)
 		throw Error(format("Useless call to computeFSClosure, at leat withComponents or withState must be true"));
 	
-	//TODO HOW CAN THESE PATHS ALREADY BE VALID SOMETIMES ..... ?????????????????????
 	for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
 		if ( !isValidPathTxn(txn, *i) && !isValidStatePathTxn(txn, *i) )
 			throw Error(format("Not a state or store path: ") % *i);
 	
-    //if withState is false, we filter out all state paths
+    //if withComponents, we add all component paths
-	if( withComponents && !withState ){
+	if( withComponents ){
 		for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
 			if ( isValidPathTxn(txn, *i) )
 				paths.insert(*i);
 	}
-	//if withComponents is false, we filter out all component paths
+	
-	else if( !withComponents && withState ){
+	//if withState, we add all state paths
+	if( withState ){
 		for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
 			if ( isValidStatePathTxn(txn, *i) )
 				paths.insert(*i);
 	}
-	//all
-	else{
-		paths = allPaths;	
-	}
 }
 
 void computeFSClosureRecTxn(const Transaction & txn, const Path & path, PathSet & paths, const int revision, const bool & flipDirection)

src/libstore/store-state.cc

@@ -399,7 +399,7 @@ void scanAndUpdateAllReferencesRecusivelyTxn(const Transaction & txn, const Path
 	}
 }
 
-// ******************************************************************* DB FUNCTIONS
+// ******************************************************* DB FUNCTIONS ********************************************************************
 
 /* State specific db functions */
 

src/libutil/util.cc

@@ -1216,6 +1216,19 @@ string getCallingUserName()
 }
 */
 
+/* adds the second PathSet after the first, but removing doubles from the second (union)
+ * (We assume the first PathSet has no duplicates)
+ * UNTESTED !!!!!!!!!!!!!!
+ */ 
+void pathSets_union_ordered(PathSet & originalPaths, const PathSet & newPaths)
+{
+	for (PathSet::iterator i = newPaths.begin(); i != newPaths.end(); ++i){
+		if (originalPaths.find(*i) != originalPaths.end())
+			continue;
+		originalPaths.insert(*i);
+	}
+}
+
 //merges two PathSets into one, removing doubles (union)
 PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2)
 {
@@ -1225,11 +1238,11 @@ PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2)
   	
   	set_union(vector1.begin(), vector1.end(),vector2.begin(), vector2.end(), back_inserter(setResult)); 	//Also available: set_symmetric_difference and set_intersection
 
-	PathSet diff;	
+	PathSet unionPaths;	
 	for(unsigned int i=0; i<setResult.size(); i++)
-		diff.insert(setResult[i]);
+		unionPaths.insert(setResult[i]);
 
-    return diff;
+    return unionPaths;
 }
 
 void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths)

src/libutil/util.hh

@@ -314,9 +314,12 @@ unsigned int getTimeStamp();
 
 //string getCallingUserName();
 
-/* TODO */
+/* Merges two PathSets into one, removing doubles (union) */
 PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2);
 
+/* TODO UNTESTED !!!!!!!!!!!!!! */
+void pathSets_union_ordered(PathSet & originalPaths, const PathSet & newPaths);
+
 /* TODO */
 void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths);
 

src/nix-state/nix-state.cc

@@ -437,20 +437,6 @@ void run(Strings args)
 	printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/store/hbxqq4d67j2y21xzp7yp01qjfkcjjbc7-hellohardcodedstateworld-1.0"));
 	printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/state/0qhlpz1ji4gvg3j6nk5vkcddmi3m5x1r-hellohardcodedstateworld-1.0-test2"));
 
-	PathSet p1;
-	PathSet p2;
-	PathSet p3;
-	PathSet p4;
-	p1.insert("a");
-	p1.insert("c");		//old
-	p1.insert("b");
-	p2.insert("b");
-	p2.insert("a");
-	p2.insert("cc");	//new
-	p2.insert("x");		//new
-	pathSets_difference(p1,p2,p3,p4);
-	pathSets_union(p1,p2);
-
 	store->scanForAllReferences("/nix/state/i06flm2ahq5s0x3633z30dnav9f1wkb5-hellohardcodedstateworld-dep1-1.0-test");
 
 	store = openStore();
@@ -575,18 +561,32 @@ void run(Strings args)
 	
 	copyContents("/nix/state/fwir6jlqygy90zadnx95zryfa8918qac-hellohardcodedstateworld-1.0-test/", "/home/wouterdb/tmp/aa/");  //TODO !!!!!!!!!!!!!!!!!!!
 	
-	
-	
-	store = openStore();
 	Hash hash;
 	PathSet references;
 	PathSet stateReferences;
 	registerValidPath(noTxn, "/nix/store/sjnaisdpqyckc75c3mjz4msi9s1662cw-hellohardcodedstateworld-1.0", hash, references, stateReferences, "/nix/store/y7abzzpiknzps5kjb4qvdxvlhvxl6slj-hellohardcodedstateworld_import-1.0.drv", 0);
+	
+	store = openStore();
+	store->isStateComponent("/nix/state/h72n75fx3c40aslbbp41vmj9gxl2skp1-hellohardcodedstateworld-1.0-test");
+	
+	PathSet p1;
+	PathSet p2;
+	PathSet p3;
+	PathSet p4;
+	p1.insert("a");
+	p1.insert("c");		//old
+	p1.insert("b");
+	p2.insert("b");
+	p2.insert("a");
+	p2.insert("cc");	//new
+	p2.insert("x");		//new
+	//pathSets_difference(p1,p2,p3,p4);
+	pathSets_union_ordered(p1,p2);
+	
+	throw Error(format("aaa"));
 	return;
 
-	//*/
+	*/
-
-	
 	
 	/* test */
 	

src/nix-store/help.txt

@@ -21,6 +21,7 @@ Operations:
 
   --init: initialise the Nix database
   --verify: verify Nix structures
+  --optimise: optimise the Nix store by hard-linking identical files
 
   --version: output version information
   --help: display help

src/nix-store/nix-store.cc

@@ -470,6 +470,13 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
 }
 
 
+static string showBytes(unsigned long long bytes)
+{
+    return (format("%d bytes (%.2f MiB)")
+        % bytes % (bytes / (1024.0 * 1024.0))).str();
+}
+
+
 struct PrintFreed 
 {
     bool show, dryRun;
@@ -481,9 +488,9 @@ struct PrintFreed
         if (show)
             cout << format(
                 (dryRun
-                    ? "%d bytes would be freed (%.2f MiB)\n"
+                    ? "%1% would be freed\n"
-                    : "%d bytes freed (%.2f MiB)\n"))
+                    : "%1% freed\n"))
-                % bytesFreed % (bytesFreed / (1024.0 * 1024.0));
+                % showBytes(bytesFreed);
     }
 };
 
@@ -618,6 +625,46 @@ static void opVerify(Strings opFlags, Strings opArgs)
 }
 
 
+
+static void showOptimiseStats(OptimiseStats & stats)
+{
+    printMsg(lvlError,
+        format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total")
+        % showBytes(stats.bytesFreed)
+        % stats.filesLinked
+        % stats.sameContents
+        % stats.totalFiles);
+}
+
+
+/* Optimise the disk space usage of the Nix store by hard-linking
+   files with the same contents. */
+static void opOptimise(Strings opFlags, Strings opArgs)
+{
+    if (!opArgs.empty())
+        throw UsageError("no arguments expected");
+
+    bool dryRun = false;
+
+    for (Strings::iterator i = opFlags.begin();
+         i != opFlags.end(); ++i)
+        if (*i == "--dry-run") dryRun = true;
+        else throw UsageError(format("unknown flag `%1%'") % *i);
+
+    LocalStore * store2(dynamic_cast<LocalStore *>(store.get()));
+    if (!store2) throw Error("you don't have sufficient rights to use --optimise");
+
+    OptimiseStats stats;
+    try {
+        store2->optimiseStore(dryRun, stats);
+    } catch (...) {
+        showOptimiseStats(stats);
+        throw;
+    }
+    showOptimiseStats(stats);
+}
+
+
 /* Scan the arguments; find the operation, set global flags, put all
    other flags in a list, and put all other arguments in another
    list. */
@@ -663,6 +710,8 @@ void run(Strings args)
             op = opInit;
         else if (arg == "--verify")
             op = opVerify;
+        else if (arg == "--optimise")
+            op = opOptimise;
         else if (arg == "--add-root") {
             if (i == args.end())
                 throw UsageError("`--add-root requires an argument");